In this document we present the joint analysis of the PASS1A metabolomics datasets.

Load all datasets

Load the data from the cloud, including: phenotypic data, metabolomic datasets, and metabolomics dictionary.

source("~/Desktop/repos/motrpac-bic-norm-qc/tools/supervised_normalization_functions.R")
source("~/Desktop/repos/motrpac-bic-norm-qc/tools/unsupervised_normalization_functions.R")
source("~/Desktop/repos/motrpac-bic-norm-qc/tools/gcp_functions.R")
source("~/Desktop/repos/motrpac-bic-norm-qc/tools/association_analysis_methods.R")
source("~/Desktop/repos/motrpac-bic-norm-qc/tools/data_aux_functions.R")
source("~/Desktop/repos/motrpac/tools/prediction_ml_tools.R")
library(randomForest) # for classification tests
# Load the dmaqc data
merged_dmaqc_data =  load_from_bucket("merged_dmaqc_data2019-10-15.RData",
    "gs://bic_data_analysis/pass1a/pheno_dmaqc/",F)
merged_dmaqc_data = merged_dmaqc_data[[1]]
rownames(merged_dmaqc_data) = as.character(merged_dmaqc_data$vial_label)
# define the tissue variable
merged_dmaqc_data$tissue = merged_dmaqc_data$sampletypedescription
# define the time to freeze variable
merged_dmaqc_data$time_to_freeze = merged_dmaqc_data$calculated.variables.time_death_to_collect_min + 
  merged_dmaqc_data$calculated.variables.time_collect_to_freeze_min
# col time vs. control
# df = data.frame(
#   bid = merged_dmaqc_data$bid,
#   edta_col_time = merged_dmaqc_data$calculated.variables.edta_coll_time,
#   time_to_freeze = merged_dmaqc_data$time_to_freeze,
#   is_control = merged_dmaqc_data$animal.key.is_control,
#   tp = merged_dmaqc_data$animal.key.timepoint,
#   tissue = merged_dmaqc_data$specimen.processing.sampletypedescription
# )
# df = unique(df)
# boxplot(edta_col_time/3600 ~ is_control,df)
# boxplot(edta_col_time/3600 - tp ~ is_control,df)
# wilcox.test(edta_col_time/3600 ~ is_control,df)
# blood freeze times
blood_samples = 
  merged_dmaqc_data$specimen.processing.sampletypedescription ==
  "EDTA Plasma"
blood_freeze_time = 
  as.difftime(merged_dmaqc_data$specimen.processing.t_freeze,units = "mins") -
  as.difftime(merged_dmaqc_data$specimen.processing.t_edtaspin,units="mins")
blood_freeze_time = as.numeric(blood_freeze_time)
time_to_freeze = merged_dmaqc_data$time_to_freeze[blood_samples] = 
  blood_freeze_time[blood_samples]
# Load our parsed metabolomics datasets
metabolomics_parsed_datasets = load_from_bucket(
  file = "metabolomics_parsed_datasets_pass1a_external1.RData",
  bucket = "gs://bic_data_analysis/pass1a/metabolomics/")[[1]]
# # Read the dictionary
# dict_bucket = 
#   "gs://motrpac-external-release1-results/metabolomics_targeted/motrpac_metabolomics_data_dictionary-v1.1.5.txt"
# dict_download = get_single_file_from_bucket_to_local_dir(dict_bucket)
# metabolomics_dict = fread(dict_download[[1]],data.table = F)
# Plot cv vs means
library(gplots)
d = metabolomics_parsed_datasets[["white_adipose_powder,metab_u_hilicpos,unnamed"]]
dx = d$sample_data
CoV<-function(x){return(sd(x,na.rm = T)/mean(x,na.rm=T))}
dmeans = apply(dx,1,mean,na.rm=T)
CoVs = apply(dx,1,CoV)
inds = !is.na(CoVs)
df = data.frame(Mean_intensity = dmeans[inds],CoV = CoVs[inds])
plot(CoV~Mean_intensity,df,cex=0.5,pch=20)
lines(lowess(CoV~Mean_intensity,df),lty=2,lwd=2,col="blue")

dx = log(1+d$sample_data,base=2)
dmeans = apply(dx,1,mean,na.rm=T)
CoVs = apply(dx,1,CoV)
inds = !is.na(CoVs)
df = data.frame(Mean_intensity = dmeans[inds],CoV = CoVs[inds])
plot(CoV~Mean_intensity,df,cex=0.5,pch=20)
lines(lowess(CoV~Mean_intensity,df),lty=2,lwd=2,col="blue")

# Plot number of NAs vs intensity mean
dx = log(1+d$sample_data,base=2)
dmeans = apply(dx,1,mean,na.rm=T)
num_nas = rowSums(is.na(dx))
df = data.frame(Num_NAs = num_nas[inds],Mean_intensity = dmeans[inds])
plot(Num_NAs~Mean_intensity,df,cex=0.5,pch=20)
lines(lowess(Num_NAs~Mean_intensity,df),lty=2,lwd=2,col="blue")

cor(df$Num_NAs,df$Mean_intensity,method="spearman")

Define the variables to be adjusted for:

biospec_cols = c(
  "acute.test.distance",
  "calculated.variables.time_to_freeze",
  # "calculated.variables.edta_coll_time", # no need - see code above for blood
  "bid" # required for matching datasets
  )
differential_analysis_cols = c(
  "animal.registration.sex",
  "animal.key.timepoint",
  "animal.key.is_control"
)
pipeline_qc_cols = c("sample_order")

Data filtering and normalization

We go over each dataset and merge the named and unnamed subparts of the untargeted datasets. For targeted data - the release buckets we merge datasets that are from the same site and tissue.

Then the preprocessing of the new datasets is as follows: (1) log the data (values are raw intensities), (2) remove rows with \(>20\%\) missing values, and (3) impute missing values.

Important: datasets that do not have unique metabolite names or sample ids probably had errors while parsing and are therefore ignored. Also, untargeted with failed mergning of the unnamed and named subsets (e.g., due to different sample ids) are ignored as well.

Finally we add an additional version for each dataset by directly regressing out the sample order component. See https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4757603/ for more details (trend correction is also called background correction, and this is done for each batch separately).

save_to_bucket(metabolomics_processed_datasets,
               file="metabolomics_processed_datasets10282019.RData",
               bucket = "gs://bic_data_analysis/pass1a/metabolomics/")
Copying file://metabolomics_processed_datasets10282019.RData [Content-Type=application/octet-stream]...
/ [0 files][    0.0 B/236.4 MiB]                                                
==> NOTE: You are uploading one or more large file(s), which would run
significantly faster if you enable parallel composite uploads. This
feature can be enabled by editing the
"parallel_composite_upload_threshold" value in your .boto
configuration file. However, note that if you do this large files will
be uploaded as `composite objects
<https://cloud.google.com/storage/docs/composite-objects>`_,which
means that any user who downloads such objects will need to have a
compiled crcmod installed (see "gsutil help crcmod"). This is because
without a compiled crcmod, computing checksums on composite objects is
so slow that gsutil disables downloads of composite objects.

-
- [0 files][792.0 KiB/236.4 MiB]                                                
\
\ [0 files][  1.6 MiB/236.4 MiB]                                                
|
/
/ [0 files][  2.3 MiB/236.4 MiB]                                                
-
- [0 files][  2.8 MiB/236.4 MiB]                                                
\
|
| [0 files][  3.6 MiB/236.4 MiB]                                                
/
/ [0 files][  4.4 MiB/236.4 MiB]                                                
-
\
\ [0 files][  5.2 MiB/236.4 MiB]                                                
|
| [0 files][  5.9 MiB/236.4 MiB]                                                
/
-
- [0 files][  6.7 MiB/236.4 MiB]                                                
\
\ [0 files][  7.5 MiB/236.4 MiB]  584.3 KiB/s                                   
|
/
/ [0 files][  8.3 MiB/236.4 MiB]  595.5 KiB/s                                   
-
- [0 files][  9.0 MiB/236.4 MiB]  615.9 KiB/s                                   
\
|
| [0 files][  9.8 MiB/236.4 MiB]  643.9 KiB/s                                   
/
/ [0 files][ 10.6 MiB/236.4 MiB]  724.6 KiB/s                                   
-
\
\ [0 files][ 11.3 MiB/236.4 MiB]  705.5 KiB/s                                   
|
| [0 files][ 11.9 MiB/236.4 MiB]  659.8 KiB/s                                   
/
-
- [0 files][ 12.4 MiB/236.4 MiB]  568.8 KiB/s                                   
\
\ [0 files][ 13.2 MiB/236.4 MiB]  561.7 KiB/s                                   
|
/
/ [0 files][ 13.9 MiB/236.4 MiB]  564.0 KiB/s                                   
-
- [0 files][ 14.7 MiB/236.4 MiB]  601.3 KiB/s                                   
\
|
| [0 files][ 15.5 MiB/236.4 MiB]  690.7 KiB/s                                   
/
/ [0 files][ 16.2 MiB/236.4 MiB]  697.3 KiB/s                                   
-
\
\ [0 files][ 17.0 MiB/236.4 MiB]  697.6 KiB/s                                   
|
| [0 files][ 17.8 MiB/236.4 MiB]  697.0 KiB/s                                   
/
-
- [0 files][ 18.6 MiB/236.4 MiB]  698.4 KiB/s                                   
\
\ [0 files][ 19.3 MiB/236.4 MiB]  701.8 KiB/s                                   
|
/
/ [0 files][ 20.1 MiB/236.4 MiB]  699.6 KiB/s                                   
-
- [0 files][ 20.9 MiB/236.4 MiB]  701.1 KiB/s                                   
\
|
| [0 files][ 21.7 MiB/236.4 MiB]  694.5 KiB/s                                   
/
/ [0 files][ 22.4 MiB/236.4 MiB]  695.0 KiB/s                                   
-
\
\ [0 files][ 23.2 MiB/236.4 MiB]  695.0 KiB/s                                   
|
| [0 files][ 24.0 MiB/236.4 MiB]  710.8 KiB/s                                   
/
-
- [0 files][ 24.8 MiB/236.4 MiB]  718.3 KiB/s                                   
\
\ [0 files][ 25.5 MiB/236.4 MiB]  717.3 KiB/s                                   
|
/
/ [0 files][ 26.3 MiB/236.4 MiB]  709.0 KiB/s                                   
-
- [0 files][ 27.1 MiB/236.4 MiB]  707.2 KiB/s                                   
\
|
| [0 files][ 27.8 MiB/236.4 MiB]  707.8 KiB/s                                   
/
/ [0 files][ 28.6 MiB/236.4 MiB]  699.6 KiB/s                                   
-
\
\ [0 files][ 29.4 MiB/236.4 MiB]  714.4 KiB/s                                   
|
| [0 files][ 30.2 MiB/236.4 MiB]  716.4 KiB/s                                   
/
-
- [0 files][ 30.9 MiB/236.4 MiB]  720.3 KiB/s                                   
\
\ [0 files][ 31.7 MiB/236.4 MiB]  723.4 KiB/s                                   
|
/
/ [0 files][ 32.5 MiB/236.4 MiB]  707.3 KiB/s                                   
-
- [0 files][ 33.3 MiB/236.4 MiB]  706.9 KiB/s                                   
\
|
| [0 files][ 34.0 MiB/236.4 MiB]  713.6 KiB/s                                   
/
/ [0 files][ 34.8 MiB/236.4 MiB]  704.6 KiB/s                                   
-
\
\ [0 files][ 35.6 MiB/236.4 MiB]  695.8 KiB/s                                   
|
| [0 files][ 36.4 MiB/236.4 MiB]  697.3 KiB/s                                   
/
-
- [0 files][ 37.1 MiB/236.4 MiB]  690.9 KiB/s                                   
\
\ [0 files][ 37.9 MiB/236.4 MiB]  689.2 KiB/s                                   
|
/
/ [0 files][ 38.7 MiB/236.4 MiB]  688.3 KiB/s                                   
-
- [0 files][ 39.5 MiB/236.4 MiB]  683.8 KiB/s                                   
\
|
| [0 files][ 40.2 MiB/236.4 MiB]  613.0 KiB/s                                   
/
/ [0 files][ 40.7 MiB/236.4 MiB]  499.7 KiB/s                                   
-
\
\ [0 files][ 41.5 MiB/236.4 MiB]  441.9 KiB/s                                   
|
/
/ [0 files][ 42.0 MiB/236.4 MiB]  416.6 KiB/s                                   
-
\
\ [0 files][ 42.5 MiB/236.4 MiB]  440.8 KiB/s                                   
|
| [0 files][ 43.1 MiB/236.4 MiB]  398.9 KiB/s                                   
/
/ [0 files][ 43.6 MiB/236.4 MiB]  415.6 KiB/s                                   
-
- [0 files][ 44.1 MiB/236.4 MiB]  428.6 KiB/s                                   
\
\ [0 files][ 44.6 MiB/236.4 MiB]  443.4 KiB/s                                   
|
/
/ [0 files][ 45.4 MiB/236.4 MiB]  498.4 KiB/s                                   
-
- [0 files][ 46.2 MiB/236.4 MiB]  568.0 KiB/s                                   
\
|
| [0 files][ 46.9 MiB/236.4 MiB]  623.1 KiB/s                                   
/
/ [0 files][ 47.7 MiB/236.4 MiB]  670.7 KiB/s                                   
-
\
\ [0 files][ 48.5 MiB/236.4 MiB]  700.1 KiB/s                                   
|
| [0 files][ 49.2 MiB/236.4 MiB]  696.4 KiB/s                                   
/
-
- [0 files][ 50.0 MiB/236.4 MiB]  685.0 KiB/s                                   
\
\ [0 files][ 50.8 MiB/236.4 MiB]  687.2 KiB/s                                   
|
/
/ [0 files][ 51.6 MiB/236.4 MiB]  676.6 KiB/s                                   
-
- [0 files][ 52.3 MiB/236.4 MiB]  681.6 KiB/s                                   
\
|
| [0 files][ 52.9 MiB/236.4 MiB]  637.2 KiB/s                                   
/
/ [0 files][ 53.4 MiB/236.4 MiB]  582.5 KiB/s                                   
-
- [0 files][ 54.1 MiB/236.4 MiB]  546.8 KiB/s                                   
\
|
| [0 files][ 54.9 MiB/236.4 MiB]  545.0 KiB/s                                   
/
/ [0 files][ 55.7 MiB/236.4 MiB]  609.1 KiB/s                                   
-
\
\ [0 files][ 56.5 MiB/236.4 MiB]  677.9 KiB/s                                   
|
| [0 files][ 57.2 MiB/236.4 MiB]  704.3 KiB/s                                   
/
-
- [0 files][ 58.0 MiB/236.4 MiB]  715.2 KiB/s                                   
\
\ [0 files][ 58.8 MiB/236.4 MiB]  719.1 KiB/s                                   
|
/
/ [0 files][ 59.6 MiB/236.4 MiB]  715.7 KiB/s                                   
-
- [0 files][ 60.3 MiB/236.4 MiB]  694.8 KiB/s                                   
\
|
| [0 files][ 61.1 MiB/236.4 MiB]  704.6 KiB/s                                   
/
/ [0 files][ 61.9 MiB/236.4 MiB]  691.8 KiB/s                                   
-
\
\ [0 files][ 62.7 MiB/236.4 MiB]  692.8 KiB/s                                   
|
| [0 files][ 63.4 MiB/236.4 MiB]  693.7 KiB/s                                   
/
-
- [0 files][ 64.2 MiB/236.4 MiB]  711.6 KiB/s                                   
\
\ [0 files][ 65.0 MiB/236.4 MiB]  707.4 KiB/s                                   
|
/
/ [0 files][ 65.7 MiB/236.4 MiB]  709.9 KiB/s                                   
-
- [0 files][ 66.5 MiB/236.4 MiB]  703.2 KiB/s                                   
\
|
| [0 files][ 67.3 MiB/236.4 MiB]  703.4 KiB/s                                   
/
/ [0 files][ 68.1 MiB/236.4 MiB]  706.5 KiB/s                                   
-
\
\ [0 files][ 68.8 MiB/236.4 MiB]  697.6 KiB/s                                   
|
| [0 files][ 69.6 MiB/236.4 MiB]  701.8 KiB/s                                   
/
-
- [0 files][ 70.4 MiB/236.4 MiB]  708.6 KiB/s                                   
\
\ [0 files][ 71.2 MiB/236.4 MiB]  704.0 KiB/s                                   
|
/
/ [0 files][ 71.9 MiB/236.4 MiB]  695.6 KiB/s                                   
-
- [0 files][ 72.7 MiB/236.4 MiB]  692.6 KiB/s                                   
\
|
| [0 files][ 73.5 MiB/236.4 MiB]  692.2 KiB/s                                   
/
/ [0 files][ 74.3 MiB/236.4 MiB]  695.8 KiB/s                                   
-
\
\ [0 files][ 75.0 MiB/236.4 MiB]  707.5 KiB/s                                   
|
| [0 files][ 75.8 MiB/236.4 MiB]  701.9 KiB/s                                   
/
-
- [0 files][ 76.6 MiB/236.4 MiB]  707.7 KiB/s                                   
\
\ [0 files][ 77.3 MiB/236.4 MiB]  716.4 KiB/s                                   
|
/
/ [0 files][ 78.1 MiB/236.4 MiB]  717.2 KiB/s                                   
-
- [0 files][ 78.9 MiB/236.4 MiB]  715.5 KiB/s                                   
\
|
| [0 files][ 79.7 MiB/236.4 MiB]  710.7 KiB/s                                   
/
/ [0 files][ 80.4 MiB/236.4 MiB]  712.6 KiB/s                                   
-
\
\ [0 files][ 81.2 MiB/236.4 MiB]  713.9 KiB/s                                   
|
| [0 files][ 82.0 MiB/236.4 MiB]  717.2 KiB/s                                   
/
-
- [0 files][ 82.8 MiB/236.4 MiB]  718.5 KiB/s                                   
\
\ [0 files][ 83.5 MiB/236.4 MiB]  716.3 KiB/s                                   
|
/
/ [0 files][ 84.3 MiB/236.4 MiB]  722.0 KiB/s                                   
-
- [0 files][ 85.1 MiB/236.4 MiB]  706.5 KiB/s                                   
\
|
| [0 files][ 85.9 MiB/236.4 MiB]  709.6 KiB/s                                   
/
/ [0 files][ 86.6 MiB/236.4 MiB]  707.8 KiB/s                                   
-
\
\ [0 files][ 87.4 MiB/236.4 MiB]  709.1 KiB/s                                   
|
| [0 files][ 88.2 MiB/236.4 MiB]  701.4 KiB/s                                   
/
-
- [0 files][ 89.0 MiB/236.4 MiB]  714.8 KiB/s                                   
\
\ [0 files][ 89.7 MiB/236.4 MiB]  702.0 KiB/s                                   
|
/
/ [0 files][ 90.5 MiB/236.4 MiB]  702.1 KiB/s                                   
-
- [0 files][ 91.3 MiB/236.4 MiB]  696.5 KiB/s                                   
\
|
| [0 files][ 92.0 MiB/236.4 MiB]  664.0 KiB/s                                   
/
/ [0 files][ 92.8 MiB/236.4 MiB]  668.1 KiB/s                                   
-
\
\ [0 files][ 93.6 MiB/236.4 MiB]  651.1 KiB/s                                   
|
| [0 files][ 94.4 MiB/236.4 MiB]  652.6 KiB/s                                   
/
-
- [0 files][ 95.1 MiB/236.4 MiB]  675.7 KiB/s                                   
\
\ [0 files][ 95.9 MiB/236.4 MiB]  682.9 KiB/s                                   
|
/
/ [0 files][ 96.7 MiB/236.4 MiB]  704.0 KiB/s                                   
-
- [0 files][ 97.5 MiB/236.4 MiB]  713.1 KiB/s                                   
\
|
| [0 files][ 98.2 MiB/236.4 MiB]  715.0 KiB/s                                   
/
/ [0 files][ 99.0 MiB/236.4 MiB]  704.0 KiB/s                                   
-
\
\ [0 files][ 99.8 MiB/236.4 MiB]  711.6 KiB/s                                   
|
| [0 files][100.6 MiB/236.4 MiB]  700.3 KiB/s                                   
/
-
- [0 files][101.3 MiB/236.4 MiB]  652.2 KiB/s                                   
\
|
| [0 files][102.1 MiB/236.4 MiB]  658.6 KiB/s                                   
/
/ [0 files][102.9 MiB/236.4 MiB]  648.2 KiB/s                                   
-
\
\ [0 files][103.6 MiB/236.4 MiB]  655.1 KiB/s                                   
|
| [0 files][104.4 MiB/236.4 MiB]  692.7 KiB/s                                   
/
-
- [0 files][105.2 MiB/236.4 MiB]  683.7 KiB/s                                   
\
\ [0 files][106.0 MiB/236.4 MiB]  695.2 KiB/s                                   
|
/
/ [0 files][106.7 MiB/236.4 MiB]  681.1 KiB/s                                   
-
- [0 files][107.5 MiB/236.4 MiB]  677.4 KiB/s                                   
\
|
| [0 files][108.0 MiB/236.4 MiB]  636.9 KiB/s                                   
/
/ [0 files][108.8 MiB/236.4 MiB]  626.4 KiB/s                                   
-
\
\ [0 files][109.6 MiB/236.4 MiB]  644.2 KiB/s                                   
|
| [0 files][110.3 MiB/236.4 MiB]  648.8 KiB/s                                   
/
-
- [0 files][111.1 MiB/236.4 MiB]  705.2 KiB/s                                   
\
\ [0 files][111.9 MiB/236.4 MiB]  692.5 KiB/s                                   
|
/
/ [0 files][112.7 MiB/236.4 MiB]  698.7 KiB/s                                   
-
- [0 files][113.4 MiB/236.4 MiB]  688.4 KiB/s                                   
\
|
| [0 files][114.2 MiB/236.4 MiB]  689.0 KiB/s                                   
/
/ [0 files][115.0 MiB/236.4 MiB]  699.3 KiB/s                                   
-
\
\ [0 files][115.8 MiB/236.4 MiB]  692.1 KiB/s                                   
|
| [0 files][116.5 MiB/236.4 MiB]  701.9 KiB/s                                   
/
-
- [0 files][117.3 MiB/236.4 MiB]  702.3 KiB/s                                   
\
\ [0 files][118.1 MiB/236.4 MiB]  698.0 KiB/s                                   
|
/
/ [0 files][118.9 MiB/236.4 MiB]  707.2 KiB/s                                   
-
- [0 files][119.6 MiB/236.4 MiB]  706.4 KiB/s                                   
\
|
| [0 files][120.4 MiB/236.4 MiB]  717.3 KiB/s                                   
/
/ [0 files][121.2 MiB/236.4 MiB]  710.4 KiB/s                                   
-
\
\ [0 files][122.0 MiB/236.4 MiB]  701.5 KiB/s                                   
|
| [0 files][122.7 MiB/236.4 MiB]  708.7 KiB/s                                   
/
-
- [0 files][123.5 MiB/236.4 MiB]  708.1 KiB/s                                   
\
\ [0 files][124.3 MiB/236.4 MiB]  709.2 KiB/s                                   
|
/
/ [0 files][125.0 MiB/236.4 MiB]  715.2 KiB/s                                   
-
- [0 files][125.8 MiB/236.4 MiB]  716.7 KiB/s                                   
\
|
| [0 files][126.6 MiB/236.4 MiB]  720.7 KiB/s                                   
/
/ [0 files][127.4 MiB/236.4 MiB]  705.5 KiB/s                                   
-
\
\ [0 files][128.1 MiB/236.4 MiB]  703.0 KiB/s                                   
|
| [0 files][128.9 MiB/236.4 MiB]  705.7 KiB/s                                   
/
-
- [0 files][129.7 MiB/236.4 MiB]  689.2 KiB/s                                   
\
\ [0 files][130.5 MiB/236.4 MiB]  707.1 KiB/s                                   
|
/
/ [0 files][131.2 MiB/236.4 MiB]  700.2 KiB/s                                   
-
- [0 files][132.0 MiB/236.4 MiB]  698.2 KiB/s                                   
\
|
| [0 files][132.8 MiB/236.4 MiB]  700.4 KiB/s                                   
/
/ [0 files][133.6 MiB/236.4 MiB]  711.4 KiB/s                                   
-
\
\ [0 files][134.3 MiB/236.4 MiB]  708.8 KiB/s                                   
|
| [0 files][135.1 MiB/236.4 MiB]  706.8 KiB/s                                   
/
-
- [0 files][135.9 MiB/236.4 MiB]  705.8 KiB/s                                   
\
\ [0 files][136.6 MiB/236.4 MiB]  708.4 KiB/s                                   
|
/
/ [0 files][137.4 MiB/236.4 MiB]  709.5 KiB/s                                   
-
- [0 files][138.2 MiB/236.4 MiB]  716.4 KiB/s                                   
\
|
| [0 files][139.0 MiB/236.4 MiB]  718.1 KiB/s                                   
/
/ [0 files][139.7 MiB/236.4 MiB]  710.1 KiB/s                                   
-
\
\ [0 files][140.5 MiB/236.4 MiB]  707.6 KiB/s                                   
|
| [0 files][141.3 MiB/236.4 MiB]  716.8 KiB/s                                   
/
-
- [0 files][142.1 MiB/236.4 MiB]  684.2 KiB/s                                   
\
\ [0 files][142.8 MiB/236.4 MiB]  687.6 KiB/s                                   
|
/
/ [0 files][143.6 MiB/236.4 MiB]  691.2 KiB/s                                   
-
- [0 files][144.4 MiB/236.4 MiB]  689.4 KiB/s                                   
\
|
| [0 files][145.2 MiB/236.4 MiB]  692.5 KiB/s                                   
/
/ [0 files][145.9 MiB/236.4 MiB]  699.6 KiB/s                                   
-
\
\ [0 files][146.7 MiB/236.4 MiB]  705.8 KiB/s                                   
|
| [0 files][147.5 MiB/236.4 MiB]  706.6 KiB/s                                   
/
/ [0 files][148.0 MiB/236.4 MiB]  617.0 KiB/s                                   
-
\
\ [0 files][148.5 MiB/236.4 MiB]  484.8 KiB/s                                   
|
/
/ [0 files][149.3 MiB/236.4 MiB]  482.3 KiB/s                                   
-
- [0 files][150.1 MiB/236.4 MiB]  492.7 KiB/s                                   
\
|
| [0 files][150.8 MiB/236.4 MiB]  582.4 KiB/s                                   
/
/ [0 files][151.6 MiB/236.4 MiB]  712.8 KiB/s                                   
-
\
\ [0 files][152.4 MiB/236.4 MiB]  710.2 KiB/s                                   
|
| [0 files][153.1 MiB/236.4 MiB]  710.7 KiB/s                                   
/
-
- [0 files][153.9 MiB/236.4 MiB]  707.4 KiB/s                                   
\
\ [0 files][154.7 MiB/236.4 MiB]  711.6 KiB/s                                   
|
/
/ [0 files][155.5 MiB/236.4 MiB]  711.8 KiB/s                                   
-
- [0 files][156.2 MiB/236.4 MiB]  690.1 KiB/s                                   
\
|
| [0 files][157.0 MiB/236.4 MiB]  697.7 KiB/s                                   
/
/ [0 files][157.8 MiB/236.4 MiB]  692.5 KiB/s                                   
-
\
\ [0 files][158.6 MiB/236.4 MiB]  687.6 KiB/s                                   
|
| [0 files][159.3 MiB/236.4 MiB]  703.6 KiB/s                                   
/
-
- [0 files][160.1 MiB/236.4 MiB]  706.4 KiB/s                                   
\
\ [0 files][160.9 MiB/236.4 MiB]  717.4 KiB/s                                   
|
/
/ [0 files][161.7 MiB/236.4 MiB]  715.4 KiB/s                                   
-
- [0 files][162.4 MiB/236.4 MiB]  708.1 KiB/s                                   
\
|
| [0 files][163.2 MiB/236.4 MiB]  710.4 KiB/s                                   
/
/ [0 files][164.0 MiB/236.4 MiB]  709.6 KiB/s                                   
-
\
\ [0 files][164.7 MiB/236.4 MiB]  699.7 KiB/s                                   
|
| [0 files][165.5 MiB/236.4 MiB]  724.1 KiB/s                                   
/
-
- [0 files][166.3 MiB/236.4 MiB]  716.4 KiB/s                                   
\
\ [0 files][167.1 MiB/236.4 MiB]  722.8 KiB/s                                   
|
/
/ [0 files][167.8 MiB/236.4 MiB]  711.3 KiB/s                                   
-
- [0 files][168.6 MiB/236.4 MiB]  696.3 KiB/s                                   
\
|
| [0 files][169.4 MiB/236.4 MiB]  700.8 KiB/s                                   
/
/ [0 files][170.2 MiB/236.4 MiB]  695.2 KiB/s                                   
-
\
\ [0 files][170.9 MiB/236.4 MiB]  698.5 KiB/s                                   
|
| [0 files][171.7 MiB/236.4 MiB]  712.5 KiB/s                                   
/
-
- [0 files][172.5 MiB/236.4 MiB]  709.0 KiB/s                                   
\
\ [0 files][173.3 MiB/236.4 MiB]  718.5 KiB/s                                   
|
/
/ [0 files][174.0 MiB/236.4 MiB]  706.6 KiB/s                                   
-
- [0 files][174.8 MiB/236.4 MiB]  715.6 KiB/s                                   
\
|
| [0 files][175.6 MiB/236.4 MiB]  716.4 KiB/s                                   
/
/ [0 files][176.3 MiB/236.4 MiB]  713.3 KiB/s                                   
-
\
\ [0 files][177.1 MiB/236.4 MiB]  709.5 KiB/s                                   
|
| [0 files][177.9 MiB/236.4 MiB]  719.8 KiB/s                                   
/
-
- [0 files][178.7 MiB/236.4 MiB]  718.8 KiB/s                                   
\
\ [0 files][179.4 MiB/236.4 MiB]  713.9 KiB/s                                   
|
/
/ [0 files][180.2 MiB/236.4 MiB]  708.2 KiB/s                                   
-
- [0 files][181.0 MiB/236.4 MiB]  700.1 KiB/s                                   
\
|
| [0 files][181.8 MiB/236.4 MiB]  710.3 KiB/s                                   
/
/ [0 files][182.5 MiB/236.4 MiB]  706.0 KiB/s                                   
-
\
\ [0 files][183.3 MiB/236.4 MiB]  701.7 KiB/s                                   
|
| [0 files][184.1 MiB/236.4 MiB]  709.6 KiB/s                                   
/
-
- [0 files][184.9 MiB/236.4 MiB]  704.7 KiB/s                                   
\
\ [0 files][185.6 MiB/236.4 MiB]  696.4 KiB/s                                   
|
/
/ [0 files][186.4 MiB/236.4 MiB]  689.6 KiB/s                                   
-
- [0 files][187.2 MiB/236.4 MiB]  687.6 KiB/s                                   
\
|
| [0 files][188.0 MiB/236.4 MiB]  665.9 KiB/s                                   
/
/ [0 files][188.5 MiB/236.4 MiB]  536.5 KiB/s                                   
-
- [0 files][189.0 MiB/236.4 MiB]  492.5 KiB/s                                   
\
|
| [0 files][189.8 MiB/236.4 MiB]  480.4 KiB/s                                   
/
/ [0 files][190.5 MiB/236.4 MiB]  517.4 KiB/s                                   
-
\
\ [0 files][191.3 MiB/236.4 MiB]  625.5 KiB/s                                   
|
| [0 files][192.1 MiB/236.4 MiB]  674.9 KiB/s                                   
/
-
- [0 files][192.8 MiB/236.4 MiB]  716.5 KiB/s                                   
\
\ [0 files][193.6 MiB/236.4 MiB]  710.8 KiB/s                                   
|
/
/ [0 files][194.4 MiB/236.4 MiB]  697.0 KiB/s                                   
-
- [0 files][195.2 MiB/236.4 MiB]  700.0 KiB/s                                   
\
|
| [0 files][195.9 MiB/236.4 MiB]  704.5 KiB/s                                   
/
/ [0 files][196.7 MiB/236.4 MiB]  706.0 KiB/s                                   
-
\
\ [0 files][197.5 MiB/236.4 MiB]  700.0 KiB/s                                   
|
| [0 files][198.3 MiB/236.4 MiB]  700.1 KiB/s                                   
/
-
- [0 files][199.0 MiB/236.4 MiB]  701.3 KiB/s                                   
\
\ [0 files][199.8 MiB/236.4 MiB]  697.6 KiB/s                                   
|
/
/ [0 files][200.6 MiB/236.4 MiB]  701.9 KiB/s                                   
-
- [0 files][201.4 MiB/236.4 MiB]  703.8 KiB/s                                   
\
|
| [0 files][202.1 MiB/236.4 MiB]  696.0 KiB/s                                   
/
/ [0 files][202.9 MiB/236.4 MiB]  704.8 KiB/s                                   
-
\
\ [0 files][203.7 MiB/236.4 MiB]  700.3 KiB/s                                   
|
| [0 files][204.5 MiB/236.4 MiB]  696.0 KiB/s                                   
/
-
- [0 files][205.2 MiB/236.4 MiB]  702.5 KiB/s                                   
\
\ [0 files][206.0 MiB/236.4 MiB]  698.0 KiB/s                                   
|
/
/ [0 files][206.8 MiB/236.4 MiB]  640.4 KiB/s                                   
-
- [0 files][207.3 MiB/236.4 MiB]  498.1 KiB/s                                   
\
|
| [0 files][207.8 MiB/236.4 MiB]  233.8 KiB/s                                   
/
/ [0 files][208.1 MiB/236.4 MiB]  189.7 KiB/s                                   
-
- [0 files][208.3 MiB/236.4 MiB]  123.7 KiB/s                                   
\
\ [0 files][208.6 MiB/236.4 MiB]  130.0 KiB/s                                   
|
/
/ [0 files][209.1 MiB/236.4 MiB]  208.3 KiB/s                                   
-
- [0 files][209.6 MiB/236.4 MiB]  223.0 KiB/s                                   
\
\ [0 files][209.9 MiB/236.4 MiB]  228.4 KiB/s                                   
|
| [0 files][210.4 MiB/236.4 MiB]  211.9 KiB/s                                   
/
/ [0 files][210.6 MiB/236.4 MiB]  238.9 KiB/s                                   
-
\
\ [0 files][211.2 MiB/236.4 MiB]  291.5 KiB/s                                   
|
| [0 files][211.4 MiB/236.4 MiB]  300.3 KiB/s                                   
/
-
- [0 files][211.9 MiB/236.4 MiB]  321.0 KiB/s                                   
\
\ [0 files][212.2 MiB/236.4 MiB]  279.1 KiB/s                                   
|
| [0 files][212.7 MiB/236.4 MiB]  164.4 KiB/s                                   
/
/ [0 files][213.0 MiB/236.4 MiB]  168.9 KiB/s                                   
-
- [0 files][213.5 MiB/236.4 MiB]  241.9 KiB/s                                   
\
\ [0 files][213.7 MiB/236.4 MiB]  225.1 KiB/s                                   
|
| [0 files][214.2 MiB/236.4 MiB]  287.7 KiB/s                                   
/
/ [0 files][214.8 MiB/236.4 MiB]  352.3 KiB/s                                   
-
\
\ [0 files][215.5 MiB/236.4 MiB]  512.4 KiB/s                                   
|
| [0 files][216.3 MiB/236.4 MiB]  554.0 KiB/s                                   
/
-
- [0 files][217.1 MiB/236.4 MiB]  616.0 KiB/s                                   
\
\ [0 files][217.9 MiB/236.4 MiB]  675.8 KiB/s                                   
|
/
/ [0 files][218.6 MiB/236.4 MiB]  682.7 KiB/s                                   
-
- [0 files][219.4 MiB/236.4 MiB]  685.8 KiB/s                                   
\
|
| [0 files][220.2 MiB/236.4 MiB]  681.2 KiB/s                                   
/
/ [0 files][221.0 MiB/236.4 MiB]  662.3 KiB/s                                   
-
\
\ [0 files][221.7 MiB/236.4 MiB]  673.1 KiB/s                                   
|
| [0 files][222.5 MiB/236.4 MiB]  675.1 KiB/s                                   
/
-
- [0 files][223.3 MiB/236.4 MiB]  686.9 KiB/s                                   
\
\ [0 files][224.0 MiB/236.4 MiB]  701.3 KiB/s                                   
|
/
/ [0 files][224.8 MiB/236.4 MiB]  705.9 KiB/s                                   
-
- [0 files][225.6 MiB/236.4 MiB]  702.4 KiB/s                                   
\
|
| [0 files][226.4 MiB/236.4 MiB]  703.7 KiB/s                                   
/
/ [0 files][227.1 MiB/236.4 MiB]  711.0 KiB/s                                   
-
\
\ [0 files][227.9 MiB/236.4 MiB]  709.1 KiB/s                                   
|
| [0 files][228.7 MiB/236.4 MiB]  710.7 KiB/s                                   
/
-
- [0 files][229.5 MiB/236.4 MiB]  714.9 KiB/s                                   
\
\ [0 files][230.2 MiB/236.4 MiB]  714.6 KiB/s                                   
|
/
/ [0 files][231.0 MiB/236.4 MiB]  711.7 KiB/s                                   
-
- [0 files][231.8 MiB/236.4 MiB]  711.9 KiB/s                                   
\
|
| [0 files][232.6 MiB/236.4 MiB]  712.1 KiB/s                                   
/
/ [0 files][233.3 MiB/236.4 MiB]  709.1 KiB/s                                   
-
\
\ [0 files][234.1 MiB/236.4 MiB]  718.4 KiB/s                                   
|
| [0 files][234.9 MiB/236.4 MiB]  713.3 KiB/s                                   
/
-
- [0 files][235.6 MiB/236.4 MiB]  719.2 KiB/s                                   
\
\ [0 files][236.4 MiB/236.4 MiB]  713.1 KiB/s                                   
|
| [1 files][236.4 MiB/236.4 MiB]  634.6 KiB/s                                   

Operation completed over 1 objects/236.4 MiB.                                    

Alternatively load the result from the bucket to save time:

metabolomics_processed_datasets = load_from_bucket(
  file="metabolomics_processed_datasets10282019.RData",
  bucket = "gs://bic_data_analysis/pass1a/metabolomics/"
)[[1]]

Examine the unlogged data

Untargeted data are typically logged and analyzed using linear models. On the other hand, concentration data are analyzed with the same type of models but using the original data. This raises a problem if we wish to compare exact statistics from these data. In this section we perform residual analysis for single metabolites. Our goal is to identify if concentration data behaves “normally” when not logged. The analysis below examines the residuals of the data and uses qq-plots to determine if a log transformation is indeed required.

Analysis of specific metabolites

Compare overlaps, effect sizes, and correlations within tissues. Compare targeted-untargeted pairs only.

# helper function to transform a metabolomics matrix
# to that of its motrpac compound ids
extract_metab_data_from_row_annot<-function(x,row_annot_x){
  # get the coloumn that has the row names
  int_sizes = apply(row_annot_x,2,function(x,y)length(intersect(x,y)),y=rownames(x))
  ind = which(int_sizes==max(int_sizes,na.rm = T))[1]
  row_annot_x = row_annot_x[is.element(row_annot_x[,ind],set=rownames(x)),]
  rownames(row_annot_x) = row_annot_x[,ind]
  shared = intersect(rownames(row_annot_x),rownames(x))
  x = x[shared,]
  row_annot_x = row_annot_x[shared,]
  rownames(x) = row_annot_x$motrpac_comp_name
  return(x)
}
single_metabolite_corrs = list()
single_metabolite_de = c()
named2covered_shared_metabolites = list()
for(nn1 in names(metabolomics_processed_datasets)){
  nn1_tissue = strsplit(nn1,split=",")[[1]][1]
  nn1_tissue = gsub("_powder","",nn1_tissue)
  if(grepl("untargeted",nn1)){next}
  single_metabolite_corrs[[nn1]] = list()
  named2covered_shared_metabolites[[nn1]] = NULL
  for(nn2 in names(metabolomics_processed_datasets)){
    if(nn2 == nn1){next}
    if(!grepl("untargeted",nn2)){next}
    nn2_tissue = strsplit(nn2,split=",")[[1]][1]
    nn2_tissue = gsub("_powder","",nn2_tissue)
    nn2_dataset = strsplit(nn2,split=",")[[1]][2]
    if(nn1_tissue!=nn2_tissue){next}
    print(nn2)
    # get the numeric dataset
    x = metabolomics_processed_datasets[[nn1]]$sample_data
    # x = log2(x+1)
    y = metabolomics_processed_datasets[[nn2]]$sample_data
    # print(paste("dataset1:",nn1))
    # print(range(x))
    # print(paste("dataset2:",nn2))
    # print(range(y))
    row_annot_x = metabolomics_processed_datasets[[nn1]]$row_annot
    row_annot_y = metabolomics_processed_datasets[[nn2]]$row_annot
    # transform metabolite names to the motrpac comp name
    x = extract_metab_data_from_row_annot(x,row_annot_x)
    y = extract_metab_data_from_row_annot(y,row_annot_y)
    # align the sample sets
    bid_y = merged_dmaqc_data[colnames(y),"bid"]
    bid_x = merged_dmaqc_data[colnames(x),"bid"]    
    # step 1: merge samples from the same BID
    if(length(unique(bid_x))!=length(bid_x)){
      x = aggregate_repeated_samples(x,bid_x)
    }
    else{
      colnames(x) = bid_x
    }
    if(length(unique(bid_y))!=length(bid_y)){
      y = aggregate_repeated_samples(y,bid_y)
    }else{
      colnames(y) = bid_y
    }
    # step 2: use the shared bio ids
    shared_bids = as.character(intersect(colnames(y),colnames(x)))
    x = as.matrix(x[,shared_bids])
    y = as.matrix(y[,shared_bids])
    # At this point x and y are over the same BIDs, now we add the metadata
    y_meta = unique(metabolomics_processed_datasets[[nn1]]$sample_meta_parsed)
    rownames(y_meta) = y_meta$bid
    y_meta = y_meta[shared_bids,]
    
    # get the shared matebolites
    shared_metabolites = intersect(rownames(x),rownames(y))
    shared_metabolites = na.omit(shared_metabolites)
    if(length(shared_metabolites)==0){next}
    named2covered_shared_metabolites[[nn1]] = union(
      named2covered_shared_metabolites[[nn1]],
      shared_metabolites
    )
    
    # Get statistics
    if(length(shared_metabolites)>1){
          corrs =cor(t(x[shared_metabolites,]),
                t(y[shared_metabolites,]),method = "spearman")
    }
    else{
          corrs = cor(x[shared_metabolites,],
                y[shared_metabolites,],method = "spearman")
    }
    
    # take the covariates, ignore distances
    curr_cov_cols = intersect(colnames(y_meta),biospec_cols[2])
    curr_covs = data.frame(y_meta[,curr_cov_cols])
    names(curr_covs) = curr_cov_cols
    # # scale
    # for(cov_name in names(curr_covs)){
    #   curr_covs[[cov_name]] = scale(curr_covs[[cov_name]],center = T,scale = T)
    # }
    curr_covs$sex = y_meta$animal.registration.sex
    
    # differential analysis
    for(tp in unique(y_meta$animal.key.timepoint)){
      resx = t(apply(
        matrix(x[shared_metabolites,],nrow=length(shared_metabolites)),1,
        pass1a_simple_differential_abundance,
        tps = y_meta$animal.key.timepoint,tp=tp,
        is_control = y_meta$animal.key.is_control,
        covs = curr_covs,return_model=F
      ))
      resy = t(apply(
        matrix(y[shared_metabolites,],nrow=length(shared_metabolites)),1,
        pass1a_simple_differential_abundance,
        tps = y_meta$animal.key.timepoint,tp=tp,
        is_control = y_meta$animal.key.is_control,
        covs = curr_covs,return_model=F
      ))
      # Add dataset information, time point, tissue
      added_columns = matrix(cbind(
        rep(nn1_dataset,length(shared_metabolites)),
        rep(nn2_dataset,length(shared_metabolites)),
        shared_metabolites,
        rep(tp,length(shared_metabolites)),
        rep(nn1_tissue,length(shared_metabolites))
      ),nrow=length(shared_metabolites))
      resx = cbind(resx,rep(T,nrow(resx)))
      colnames(resx)[ncol(resx)] = "is_targeted"
      resy = cbind(resy,rep(F,nrow(resy)))
      colnames(resy)[ncol(resy)] = "is_targeted"
      if(nrow(resx)>1){
        resx = cbind(added_columns[,-2],resx)
        resy = cbind(added_columns[,-1],resy)
      }
      else{
        resx = c(added_columns[,-2],resx)
        resy = c(added_columns[,-1],resy)
      }
      single_metabolite_de = rbind(single_metabolite_de,resx)
      single_metabolite_de = rbind(single_metabolite_de,resy)
    }
    
    single_metabolite_corrs[[nn1]][[nn2]] = corrs
  }
}
[1] "plasma,metab_u_hilicpos,untargeted"
[1] "plasma,metab_u_ionpneg,untargeted"
[1] "plasma,metab_u_lrpneg,untargeted"
[1] "plasma,metab_u_lrppos,untargeted"
[1] "plasma,metab_u_rpneg,untargeted"
[1] "plasma,metab_u_rppos,untargeted"
[1] "plasma,metab_u_hilicpos,untargeted"
[1] "plasma,metab_u_ionpneg,untargeted"
[1] "plasma,metab_u_lrpneg,untargeted"
[1] "plasma,metab_u_lrppos,untargeted"
[1] "plasma,metab_u_rpneg,untargeted"
[1] "plasma,metab_u_rppos,untargeted"
[1] "plasma,metab_u_hilicpos,untargeted"
[1] "plasma,metab_u_ionpneg,untargeted"
[1] "plasma,metab_u_lrpneg,untargeted"
[1] "plasma,metab_u_lrppos,untargeted"
[1] "plasma,metab_u_rpneg,untargeted"
[1] "plasma,metab_u_rppos,untargeted"
[1] "plasma,metab_u_hilicpos,untargeted"
[1] "plasma,metab_u_ionpneg,untargeted"
[1] "plasma,metab_u_lrpneg,untargeted"
[1] "plasma,metab_u_lrppos,untargeted"
[1] "plasma,metab_u_rpneg,untargeted"
[1] "plasma,metab_u_rppos,untargeted"
[1] "plasma,metab_u_hilicpos,untargeted"
[1] "plasma,metab_u_ionpneg,untargeted"
[1] "plasma,metab_u_lrpneg,untargeted"
[1] "plasma,metab_u_lrppos,untargeted"
[1] "plasma,metab_u_rpneg,untargeted"
[1] "plasma,metab_u_rppos,untargeted"
[1] "gastrocnemius_powder,metab_u_hilicpos,untargeted"
[1] "gastrocnemius_powder,metab_u_ionpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrppos,untargeted"
[1] "gastrocnemius_powder,metab_u_rpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_rppos,untargeted"
[1] "gastrocnemius_powder,metab_u_hilicpos,untargeted"
[1] "gastrocnemius_powder,metab_u_ionpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrppos,untargeted"
[1] "gastrocnemius_powder,metab_u_rpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_rppos,untargeted"
[1] "gastrocnemius_powder,metab_u_hilicpos,untargeted"
[1] "gastrocnemius_powder,metab_u_ionpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrppos,untargeted"
[1] "gastrocnemius_powder,metab_u_rpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_rppos,untargeted"
[1] "gastrocnemius_powder,metab_u_hilicpos,untargeted"
[1] "gastrocnemius_powder,metab_u_ionpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrppos,untargeted"
[1] "gastrocnemius_powder,metab_u_rpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_rppos,untargeted"
[1] "gastrocnemius_powder,metab_u_hilicpos,untargeted"
[1] "gastrocnemius_powder,metab_u_ionpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrppos,untargeted"
[1] "gastrocnemius_powder,metab_u_rpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_rppos,untargeted"
[1] "gastrocnemius_powder,metab_u_hilicpos,untargeted"
[1] "gastrocnemius_powder,metab_u_ionpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrppos,untargeted"
[1] "gastrocnemius_powder,metab_u_rpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_rppos,untargeted"
[1] "gastrocnemius_powder,metab_u_hilicpos,untargeted"
[1] "gastrocnemius_powder,metab_u_ionpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrppos,untargeted"
[1] "gastrocnemius_powder,metab_u_rpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_rppos,untargeted"
[1] "gastrocnemius_powder,metab_u_hilicpos,untargeted"
[1] "gastrocnemius_powder,metab_u_ionpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrppos,untargeted"
[1] "gastrocnemius_powder,metab_u_rpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_rppos,untargeted"
[1] "gastrocnemius_powder,metab_u_hilicpos,untargeted"
[1] "gastrocnemius_powder,metab_u_ionpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_lrppos,untargeted"
[1] "gastrocnemius_powder,metab_u_rpneg,untargeted"
[1] "gastrocnemius_powder,metab_u_rppos,untargeted"
[1] "liver_powder,metab_u_hilicpos,untargeted"
[1] "liver_powder,metab_u_ionpneg,untargeted"
[1] "liver_powder,metab_u_lrpneg,untargeted"
[1] "liver_powder,metab_u_lrppos,untargeted"
[1] "liver_powder,metab_u_rpneg,untargeted"
[1] "liver_powder,metab_u_rppos,untargeted"
[1] "liver_powder,metab_u_hilicpos,untargeted"
[1] "liver_powder,metab_u_ionpneg,untargeted"
[1] "liver_powder,metab_u_lrpneg,untargeted"
[1] "liver_powder,metab_u_lrppos,untargeted"
[1] "liver_powder,metab_u_rpneg,untargeted"
[1] "liver_powder,metab_u_rppos,untargeted"
[1] "liver_powder,metab_u_hilicpos,untargeted"
[1] "liver_powder,metab_u_ionpneg,untargeted"
[1] "liver_powder,metab_u_lrpneg,untargeted"
[1] "liver_powder,metab_u_lrppos,untargeted"
[1] "liver_powder,metab_u_rpneg,untargeted"
[1] "liver_powder,metab_u_rppos,untargeted"
[1] "liver_powder,metab_u_hilicpos,untargeted"
[1] "liver_powder,metab_u_ionpneg,untargeted"
[1] "liver_powder,metab_u_lrpneg,untargeted"
[1] "liver_powder,metab_u_lrppos,untargeted"
[1] "liver_powder,metab_u_rpneg,untargeted"
[1] "liver_powder,metab_u_rppos,untargeted"
[1] "liver_powder,metab_u_hilicpos,untargeted"
[1] "liver_powder,metab_u_ionpneg,untargeted"
[1] "liver_powder,metab_u_lrpneg,untargeted"
[1] "liver_powder,metab_u_lrppos,untargeted"
[1] "liver_powder,metab_u_rpneg,untargeted"
[1] "liver_powder,metab_u_rppos,untargeted"
[1] "liver_powder,metab_u_hilicpos,untargeted"
[1] "liver_powder,metab_u_ionpneg,untargeted"
[1] "liver_powder,metab_u_lrpneg,untargeted"
[1] "liver_powder,metab_u_lrppos,untargeted"
[1] "liver_powder,metab_u_rpneg,untargeted"
[1] "liver_powder,metab_u_rppos,untargeted"
[1] "liver_powder,metab_u_hilicpos,untargeted"
[1] "liver_powder,metab_u_ionpneg,untargeted"
[1] "liver_powder,metab_u_lrpneg,untargeted"
[1] "liver_powder,metab_u_lrppos,untargeted"
[1] "liver_powder,metab_u_rpneg,untargeted"
[1] "liver_powder,metab_u_rppos,untargeted"
[1] "liver_powder,metab_u_hilicpos,untargeted"
[1] "liver_powder,metab_u_ionpneg,untargeted"
[1] "liver_powder,metab_u_lrpneg,untargeted"
[1] "liver_powder,metab_u_lrppos,untargeted"
[1] "liver_powder,metab_u_rpneg,untargeted"
[1] "liver_powder,metab_u_rppos,untargeted"
[1] "white_adipose_powder,metab_u_hilicpos,untargeted"
[1] "white_adipose_powder,metab_u_lrpneg,untargeted"
[1] "white_adipose_powder,metab_u_lrppos,untargeted"
[1] "white_adipose_powder,metab_u_rpneg,untargeted"
[1] "white_adipose_powder,metab_u_rppos,untargeted"
[1] "white_adipose_powder,metab_u_hilicpos,untargeted"
[1] "white_adipose_powder,metab_u_lrpneg,untargeted"
[1] "white_adipose_powder,metab_u_lrppos,untargeted"
[1] "white_adipose_powder,metab_u_rpneg,untargeted"
[1] "white_adipose_powder,metab_u_rppos,untargeted"
[1] "white_adipose_powder,metab_u_hilicpos,untargeted"
[1] "white_adipose_powder,metab_u_lrpneg,untargeted"
[1] "white_adipose_powder,metab_u_lrppos,untargeted"
[1] "white_adipose_powder,metab_u_rpneg,untargeted"
[1] "white_adipose_powder,metab_u_rppos,untargeted"
[1] "white_adipose_powder,metab_u_hilicpos,untargeted"
[1] "white_adipose_powder,metab_u_lrpneg,untargeted"
[1] "white_adipose_powder,metab_u_lrppos,untargeted"
[1] "white_adipose_powder,metab_u_rpneg,untargeted"
[1] "white_adipose_powder,metab_u_rppos,untargeted"
[1] "white_adipose_powder,metab_u_hilicpos,untargeted"
[1] "white_adipose_powder,metab_u_lrpneg,untargeted"
[1] "white_adipose_powder,metab_u_lrppos,untargeted"
[1] "white_adipose_powder,metab_u_rpneg,untargeted"
[1] "white_adipose_powder,metab_u_rppos,untargeted"
[1] "white_adipose_powder,metab_u_hilicpos,untargeted"
[1] "white_adipose_powder,metab_u_lrpneg,untargeted"
[1] "white_adipose_powder,metab_u_lrppos,untargeted"
[1] "white_adipose_powder,metab_u_rpneg,untargeted"
[1] "white_adipose_powder,metab_u_rppos,untargeted"
[1] "white_adipose_powder,metab_u_hilicpos,untargeted"
[1] "white_adipose_powder,metab_u_lrpneg,untargeted"
[1] "white_adipose_powder,metab_u_lrppos,untargeted"
[1] "white_adipose_powder,metab_u_rpneg,untargeted"
[1] "white_adipose_powder,metab_u_rppos,untargeted"
[1] "white_adipose_powder,metab_u_hilicpos,untargeted"
[1] "white_adipose_powder,metab_u_lrpneg,untargeted"
[1] "white_adipose_powder,metab_u_lrppos,untargeted"
[1] "white_adipose_powder,metab_u_rpneg,untargeted"
[1] "white_adipose_powder,metab_u_rppos,untargeted"
[1] "white_adipose_powder,metab_u_hilicpos,untargeted"
[1] "white_adipose_powder,metab_u_lrpneg,untargeted"
[1] "white_adipose_powder,metab_u_lrppos,untargeted"
[1] "white_adipose_powder,metab_u_rpneg,untargeted"
[1] "white_adipose_powder,metab_u_rppos,untargeted"
single_metabolite_de = data.frame(single_metabolite_de)
names(single_metabolite_de) = c(
  "dataset","metabolite","tp","tissue",
  "Est","Std","Tstat","Pvalue","is_targeted")
for(col in names(single_metabolite_de)[-c(1:4)]){
  single_metabolite_de[[col]] = as.numeric(
    as.character(single_metabolite_de[[col]]))
}
for(col in names(single_metabolite_de)[1:4]){
  single_metabolite_de[[col]] = 
    as.character(single_metabolite_de[[col]])
}

We next transform the data above into tables that contain data for each combination of metabolite, time point, and tissue. These are then used for two meta-analyses: (1) a simple random effects analysis, and (2) random effects with a binary covariate indicating if a dataset is targeted or untargeted.

rownames(single_metabolite_de) = NULL
for(nn in names(single_metabolite_de)){
  ndig = 5
  if(grepl("pval",nn,ignore.case = T)){
    ndig = 10
  }
  if(is.numeric(single_metabolite_de[[nn]])){
    single_metabolite_de[[nn]] = round(single_metabolite_de[[nn]],digits = ndig)
  }
}
single_metabolite_de = unique(single_metabolite_de)
library(metafor)
meta_analysis_stats = list()
for(tissue in unique(single_metabolite_de$tissue)){
  for(tp in unique(single_metabolite_de$tp)){
    curr_subset = single_metabolite_de[
      single_metabolite_de$tissue==tissue &
        single_metabolite_de$tp==tp,]
    for(metabolite in unique(curr_subset$metabolite)){
      curr_met_data = curr_subset[
        curr_subset$metabolite==metabolite,]
      curr_met_data$var = curr_met_data$Std^2
      re_model1 = NULL;re_model2=NULL;re_model_tar = NULL
      try({re_model1 = rma.uni(curr_met_data$Est,curr_met_data$var)})
      try({re_model2 = rma.mv(curr_met_data$Est,curr_met_data$var,
                         mods=curr_met_data$is_targeted)})
      try({re_model_tar = rma.uni(
        curr_met_data[curr_met_data$is_targeted==1,"Est"],
        curr_met_data[curr_met_data$is_targeted==1,"var"]
      )})
      meta_analysis_stats[[paste(metabolite,tissue,tp,sep=",")]] = 
        list(curr_met_data=curr_met_data,re_model1=re_model1,
            re_model2 = re_model2)
    }
  }
}

We now show some plots to summarize the comparison. We plot the overall average correlation between the platforms (within tissues). However, going into a single metabolite comparison in detail (again, within tissues), for each metabolite, in each time point we perform a meta-analysis of the effects and examine the significance and heterogeneity.

# Do we have nice coverages of the named datasets?
library(ggplot2)
dataset2num_metabolites = sapply(metabolomics_processed_datasets, 
                                 function(x)nrow(x$sample_data))
named_dataset_coverage = sapply(named2covered_shared_metabolites,length)
named_dataset_coverage = data.frame(
  name = names(named_dataset_coverage),
  percentage = named_dataset_coverage /
  dataset2num_metabolites[names(named_dataset_coverage)],
  count = named_dataset_coverage,
  total = dataset2num_metabolites[names(named_dataset_coverage)]
)
# add datasets with no coverage
all_targeted_datasets = names(metabolomics_processed_datasets)
all_targeted_datasets = all_targeted_datasets[!grepl("untarg",all_targeted_datasets)]
zero_coverage_datasets = setdiff(all_targeted_datasets,
                                 named_dataset_coverage$name)
zero_coverage_datasets = data.frame(
  name = zero_coverage_datasets,
  percentage = rep(0,length(zero_coverage_datasets)),
  count = rep(0,length(zero_coverage_datasets)),
  total = dataset2num_metabolites[zero_coverage_datasets]
)
named_dataset_coverage = rbind(named_dataset_coverage,
                           zero_coverage_datasets)
named_dataset_coverage = 
  named_dataset_coverage[order(as.character(named_dataset_coverage$name)),]
print(ggplot(named_dataset_coverage, aes(x=name, y=percentage)) + 
  geom_bar(stat = "identity",width=0.2) + coord_flip() +
  geom_text(data=named_dataset_coverage, 
            aes(name, percentage+0.05, label=count), 
            position = position_dodge(width=0.9),
            size=4) + 
  ggtitle("Named datasets: coverage by untargeted datasets"))

# Next examine the Spearman correlations between platforms
extract_diag_vs_non_diag<-function(corrs,func=mean,...){
  if(length(corrs)==1){
    return(c(same=func(corrs,...),other=NA))
  }
  same = func(diag(corrs),...)
  other = func(
    c(corrs[lower.tri(corrs,diag = F)]),...)
  return(c(same=same,other=other))
}
# tar_dataset2tissue = sapply(names(single_metabolite_corrs),
#       function(x)strsplit(x,split=",")[[1]][1])
# tar_dataset2tissue = gsub("_powder","",tar_dataset2tissue)
for(tar_dataset in names(single_metabolite_corrs)){
  l = single_metabolite_corrs[[tar_dataset]]
  if(length(l)==0){next}
  corr_info = as.data.frame(t(sapply(l, extract_diag_vs_non_diag)))
  corr_sd = as.data.frame(t(sapply(l, extract_diag_vs_non_diag,func=sd)))
  rownames(corr_info) = sapply(rownames(corr_info),
      function(x)strsplit(x,split=",")[[1]][2])
  rownames(corr_info) = gsub("metab_u_","",rownames(corr_info))
  rownames(corr_sd) = rownames(corr_info)
  corr_info$dataset = rownames(corr_info)
  corr_sd$dataset = corr_info$dataset
  corr_info = melt(corr_info)
  corr_sd = melt(corr_sd)
  corr_info$sd = corr_sd$value
  print(
    ggplot(corr_info, aes(x=dataset, y=value, fill=variable)) +
      geom_bar(position=position_dodge(), stat="identity", colour='black') +
      geom_errorbar(aes(ymin=value-sd, ymax=value+sd),na.rm=T, 
                   width=.2,position=position_dodge(.9)) +
    ggtitle(tar_dataset) + xlab("Untargeted dataset") + ylab("Spearman") +
      labs(fill = "Pair type") + 
      theme(legend.position="top",legend.direction = "horizontal")
  )
}
Using dataset as id variables
Using dataset as id variables

# Look at the meta-analysis results
I2_scores = sapply(meta_analysis_stats,function(x)x$re_model1$I2)
I2_scores = unlist(I2_scores[sapply(I2_scores,length)>0])
hist(I2_scores)

betas = sapply(meta_analysis_stats,function(x)x$re_model1$beta[1,1])
betas = unlist(betas[names(I2_scores)])
pvals = sapply(meta_analysis_stats,function(x)x$re_model1$pval)
pvals = unlist(pvals[names(I2_scores)])
targeted_diff_p = 
  sapply(meta_analysis_stats,function(x)x$re_model2$pval[2])
targeted_diff_p = unlist(targeted_diff_p[names(I2_scores)])
plot(betas,I2_scores)

plot(-log10(pvals),I2_scores)

plot(-log10(targeted_diff_p),I2_scores)

plot(-log10(targeted_diff_p),-log10(pvals))

# plot examples
agree_example = names(sample(which(pvals< 0.001 & I2_scores < 20))[1])
forest(meta_analysis_stats[[agree_example]]$re_model1,
       slab = meta_analysis_stats[[agree_example]][[1]][,1],
       main = agree_example,xlab = "Log fc",
       col = "blue",cex = 1.1)

disagree_example = names(sample(which(targeted_diff_p< 0.001)))[1]
forest(meta_analysis_stats[[disagree_example]]$re_model1,
       slab = meta_analysis_stats[[disagree_example]][[1]][,1],
       main = disagree_example,xlab = "Log fc",
       col = "blue",cex = 1.1)

Dataset pairwise comparison as a prediction task

Use 10-fold cross validation for analysis within tissues.

nfolds = 10
prediction_analysis_results = list()
for(nn1 in names(metabolomics_processed_datasets)){
  nn1_tissue = strsplit(nn1,split=",")[[1]][1]
  nn1_tissue = gsub("_powder","",nn1_tissue)
  for(nn2 in names(metabolomics_processed_datasets)){
    nn2_tissue = strsplit(nn2,split=",")[[1]][1]
    nn2_tissue = gsub("_powder","",nn2_tissue)
    if(nn1_tissue!=nn2_tissue){next}
    print(paste("training set:",nn2))
    print(paste("test set:",nn1))
    
    # get the data, merge samples by bid if necessary
    y = metabolomics_processed_datasets[[nn1]]$sample_data
    x = metabolomics_processed_datasets[[nn2]]$sample_data
    y_meta = unique(metabolomics_processed_datasets[[nn1]]$sample_meta_parsed)
    # # remove metadata variables with too many NAs
    # na_counts = apply(is.na(y_meta),2,sum)
    # y_meta = y_meta[,na_counts/nrow(y_meta) == 0]
    rownames(y_meta) = y_meta$bid
    bid_y = merged_dmaqc_data[colnames(y),"bid"]
    bid_x = merged_dmaqc_data[colnames(x),"bid"]    
    # merge samples from the same BID
    if(length(unique(bid_x))!=length(bid_x)){
      x = aggregate_repeated_samples(x,bid_x)
    }
    else{
      colnames(x) = bid_x
    }
    if(length(unique(bid_y))!=length(bid_y)){
      y = aggregate_repeated_samples(y,bid_y)
    }else{
      colnames(y) = bid_y
    }
    
    if(ncol(y)>1000){next}
    
    cov_cols = intersect(colnames(y_meta),
                          setdiff(biospec_cols,"bid"))
    covs = as.matrix(y_meta[colnames(y),cov_cols])
    
    # regress the covariates out - simple linear analysis
    y = lm_regress_out_matrix(t(y),covs)
    
    # Prepare the input for the ML part
    shared_bids = as.character(intersect(rownames(y),colnames(x)))
    x = t(as.matrix(x[,shared_bids]))
    y = as.matrix(y[shared_bids,])
    
    # Run the regressions
    folds = sample(rep(1:nfolds,(1+nrow(x)/nfolds)))[1:nrow(x)]
    numFeatures = min(ncol(x),2000)
    preds = c();real=c()
    for(i in 1:ncol(y)){
      y_i = y[,1]
      i_preds = c();i_real=c()
      for(j in 1:nfolds){
        print(j)
        tr_x = x[folds!=j,]
        tr_yi = y_i[folds!=j]
        te_x = x[folds==j,]
        te_y = y_i[folds==j]
        # random forest
        # model = randomForest(tr_yi,x=tr_x,ntree = 20)
        # te_preds = predict(model,newdata = te_x)
        model = feature_selection_wrapper(tr_x,tr_yi,
                   coeff_of_var,randomForest,
                   topK = numFeatures,ntree=50)
        te_preds = predict(model,newdata = te_x)
        i_preds = c(i_preds,te_preds)
        i_real = c(i_real,te_y)
      }
      preds = cbind(preds,i_preds)
      real = cbind(real,i_real)
    }
    currname = paste(nn1,nn2,sep=";")
    prediction_analysis_results[[currname]] = list(
      preds = preds,real=real
    )
  }
}
names(prediction_analysis_results)

cov_prediction_analysis_results = list()
for(nn1 in names(metabolomics_processed_datasets)){
  print(nn1)
  y = metabolomics_processed_datasets[[nn1]]$sample_data
  y_vials = colnames(y)
  bid_y = merged_dmaqc_data[colnames(y),"bid"]
  colnames(y) = bid_y
  y = t(as.matrix(y))
  if(ncol(y)>1000){next}
  cov_cols = c("animal.registration.sex",
             "acute.test.weight",
             "acute.test.distance",
             "animal.key.timepoint")
  covs = merged_dmaqc_data[y_vials,cov_cols]
  x = covs
  
  # Run the regressions
  folds = sample(rep(1:nfolds,(1+nrow(x)/nfolds)))[1:nrow(x)]
  numFeatures = min(ncol(x),2000)
  preds = c();real=c()
  for(i in 1:ncol(y)){
    y_i = y[,1]
    i_preds = c();i_real=c()
    for(j in 1:nfolds){
      print(j)
      tr_x = x[folds!=j,]
      tr_yi = y_i[folds!=j]
      te_x = x[folds==j,]
      te_y = y_i[folds==j]
      # random forest
      model = randomForest(tr_yi,x=tr_x,ntree = 20)
      te_preds = predict(model,newdata = te_x)
      i_preds = c(i_preds,te_preds)
      i_real = c(i_real,te_y)
    }
    preds = cbind(preds,i_preds)
    real = cbind(real,i_real)
  }
  cov_prediction_analysis_results[[nn1]] = list(
      preds = preds,real=real
    )
}

results_metrics = c()
for(nn in names(prediction_analysis_results)){
  preds = prediction_analysis_results[[nn]]$preds
  real = prediction_analysis_results[[nn]]$real
  rhos1 = diag(cor(preds,real))
  rhos2 = diag(cor(preds,real,method="spearman"))
  SEs = (preds-real)^2
  mse = mean(SEs)
  normSEs = SEs / apply(real,2,sd)
  curr_scores = c(mean(rhos1),mean(rhos2),min(rhos1),min(rhos2),
                  mse,mean(normSEs))
  names(curr_scores) = c("mean_rho","mean_spearman_rho","min_rho","min_spearman_rho",
                         "MSE","mean MSE/SD")
  results_metrics = rbind(results_metrics,
                          curr_scores)
  rownames(results_metrics)[nrow(results_metrics)] = nn
}

cov_results_metrics = c()
for(nn in names(cov_prediction_analysis_results)){
  preds = cov_prediction_analysis_results[[nn]]$preds
  real = cov_prediction_analysis_results[[nn]]$real
  rhos1 = diag(cor(preds,real))
  rhos2 = diag(cor(preds,real,method="spearman"))
  SEs = (preds-real)^2
  mse = mean(SEs)
  normSEs = SEs / apply(real,2,sd)
  curr_scores = c(mean(rhos1),mean(rhos2),min(rhos1),min(rhos2),
                  mse,mean(normSEs))
  names(curr_scores) = c("mean_rho","mean_spearman_rho","min_rho","min_spearman_rho",
                         "MSE","mean MSE/SD")
  cov_results_metrics = rbind(cov_results_metrics,
                          curr_scores)
  rownames(cov_results_metrics)[nrow(cov_results_metrics)] = nn
}

# Some boxplots
pred_targets = sapply(names(prediction_analysis_results),function(x)
  strsplit(x,split = ";")[[1]][1])
target = "liver_powder,metab_t_tca,named" 
curr_res = unique(results_metrics[pred_targets == target,])
rownames(curr_res) = gsub(target,"",rownames(curr_res))
rownames(curr_res) = gsub(";","",rownames(curr_res))
rownames(curr_res)[rownames(curr_res)==""] = target
cov_baseline = cov_results_metrics[target,]

par(mar = c(8,4,2,2))
cols = rep("blue",nrow(curr_res))
cols[rownames(curr_res)==target] = "black"
plt = barplot(curr_res[,4],beside = T,xaxt="n",legend=F,
              ylab="Min rho (Spearman)",
              ylim = c(0.5,1),xpd=F,col=cols,
              main = target)
text(plt, par("usr")[3], labels = rownames(curr_res), 
     srt = 45, adj = c(1.1,1.1), xpd = T, cex=0.6)
abline(h=cov_baseline[4],lty=2,lwd=2,col="red")

par(mar = c(8,4,2,2))
plt = barplot(curr_res[,6],beside = T,xaxt="n",legend=F,
              ylab="Mean standardized SE",xpd=F,
              main = target,col=cols)
text(plt, par("usr")[3], labels = rownames(curr_res), 
     srt = 45, adj = c(1.1,1.1), xpd = T, cex=0.6)
abline(h=cov_baseline[6],lty=2,lwd=2,col="red")

# preds = c();real=c()
# for(j in 1:nfolds){
#   tr_x = x[folds!=j,]
#   tr_y = y[folds!=j,]
#   te_x = x[folds==j,]
#   te_y = y[folds==j,]
#   model = MTL_wrapper(tr_x,tr_y,type="Regression", Regularization="L21")
#   te_preds = predict(model,te_x)
#   real = rbind(real,te_y)
#   preds = rbind(preds,te_preds)
# }
# diag(cor(preds,real))

# Using PLS regression
# library(pls)
# pls_model = plsr(y~x,ncomp = 5,validation="LOO")
# eval = MSEP(pls_model)
# 
# y_pca = prcomp(y)
# plot(y_pca)
# explained_var = y_pca$sdev^2/sum(y_pca$sdev^2)
# y_pca_matrix = y_pca$x[,1:10]
# 
# # regress out sex, weight
# 
# get_explained_variance_using_PCA(x,y)
# x = apply(x,2,regress_out,covs=covs)
# y = apply(y,2,regress_out,covs=covs)
# get_explained_variance_using_PCA(x,y)

Comparison of covariance matrices


CV <-function(x){
  return(sd(x)/mean(x))
}

# Get all correlation and covariance matrices
y = metabolomics_parsed_datasets[[1]]$sample_data # anchor all datasets by these ids
y_vials = colnames(y)
bid_y = as.character(merged_dmaqc_data[colnames(y),"bid"])
cov_cols = c("animal.registration.sex",
             "acute.test.weight",
             "acute.test.distance")
covs = merged_dmaqc_data[y_vials,cov_cols]

cov_matrices = list()
cor_matrices = list()
for(nn in names(metabolomics_parsed_datasets)){
  x = metabolomics_parsed_datasets[[nn]]$sample_data
  bid_x = as.character(merged_dmaqc_data[colnames(x),"bid"])
  print(sum(!is.element(bid_y,set=bid_x))) # should be zero
  colnames(x) = bid_x
  x = x[,bid_y]
  # x = apply(x,1,regress_out,covs=covs)
  # x = t(x)
  # cvs = apply(x,1,CV)
  # print(table(cvs>0.5))
  # x = x[cvs>0.5,]
  # pcax = t(prcomp(t(x),scale. = F)$x[,1:10])
  cov_matrices[[nn]] = cov(x)
  cor_matrices[[nn]] = cor(x)
}
sapply(cor_matrices,dim)
library('evolqg');library(corrplot)
mantel_tests= MantelCor(cor_matrices)
mcors= MatrixCor(cor_matrices)
mcors = as.matrix(forceSymmetric(mcors,uplo="L"))
mantel_tests = as.matrix(
  forceSymmetric(mantel_tests$probabilities,uplo = "L"))
# diag(mantel_tests)=0
ord = corrplot(t(mcors),order="hclust")
ord = rownames(ord)
corrplot(mcors[ord,ord],p.mat=mantel_tests[ord,ord],
         insig = "label_sig",method="shade",
         sig.level = c(.0001, .001, .01),
         pch.cex = .9, pch.col = "black")
# rs_res = RandomSkewers(cor_matrices)
# corrplot(rs_res$correlations,
#          p.mat=mantel_tests$probabilities,type="lower",
#          insig = "label_sig",method="shade",
#          sig.level = c(.0001, .001, .01),
#          pch.cex = .9, pch.col = "black",tl.cex=0.6)

library(psych)

r1 = cor_matrices[[1]]
r2 = cor_matrices[[8]]
cor(r1[lower.tri(r1)],r2[lower.tri(r2)])
mantel_tests[1,8]
mantel_tests[8,1]

threshold_comp = apply_function_on_pairs(cor_matrices,
          function(x,y)sum(x>0.7&y>0.7))
corrplot(threshold_comp,is.corr = F,order="hclust")

threshold_comp = apply_function_on_pairs(cor_matrices,
         function(x,y)mean(diag(cor(x,y))))
corrplot(threshold_comp,is.corr = F,order="hclust")

Integrarion with RNAseq

rnaseq_data_for_difftests = load_from_bucket(
  "rnaseq_data_for_difftests.RData",
  "gs://bic_data_analysis/pass1a/rnaseq/"
)[[1]]
met_vs_rnaseq_cors = c()
for(nn in names(rnaseq_data_for_difftests)){
  d = rnaseq_data_for_difftests[[nn]]$fpkms
  covs = rnaseq_data_for_difftests[[nn]]$dmaqc_meta[,cov_cols]
  
  cvs = apply(d,1,CV)
  print(table(cvs>0.25))
  x = d[cvs>0.25,]
  # x = apply(x,1,regress_out,covs=covs)
  # x = t(x)
  
  bid_x = sapply(colnames(x),substr,1,5)
  print(sum(!is.element(bid_y,set=bid_x))) # should be zero
  colnames(x) = bid_x
  x = x[,bid_y]
  
  curr_cors = cor(x)
  v = c()
  for(nn2 in names(cor_matrices)){
    v[nn2] = MatrixCor(curr_cors,cor_matrices[[nn2]])
  }
  met_vs_rnaseq_cors = rbind(met_vs_rnaseq_cors,v)
  rownames(met_vs_rnaseq_cors)[nrow(met_vs_rnaseq_cors)] = nn
}
library(gplots)
heatmap.2(t(met_vs_rnaseq_cors),
          trace="none",scale=NULL,mar=c(15,15),
          key.xlab = "Correlation",
          col=redblue(200), breaks=seq(-1,1,0.01))

Looking at metabolite overlap

metabolite_sets = list()
for(nn in names(metabolomics_parsed_datasets)){
  an = metabolomics_parsed_datasets[[nn]]$row_annot
  name_cols = colnames(an)[grepl("_name",colnames(an),ignore.case = T)]
  metabolite_sets[[nn]] = unique(tolower(an[,name_cols[1]]))
}
overlap_matrix = apply_function_on_pairs(metabolite_sets,
                                         function(x,y)length(intersect(x,y)))
corrplot(log(overlap_matrix+1,10),is.corr = F)


site_cor_comparison = apply_function_on_pairs(
  metabolomics_parsed_datasets,
  compare_two_met_sites,print_progress = T,
  merged_dmaqc_data = merged_dmaqc_data,samplesize=500
)

site_cor_comparison[is.na(site_cor_comparison)]=0
corrplot(site_cor_comparison,order="hclust")

compare_two_met_sites<-function(x,y,merged_dmaqc_data,samplesize=100,...){
  an_x = x$row_annot
  name_colsx = colnames(an_x)[grepl("_name",colnames(an_x),ignore.case = T)]
  an_y = y$row_annot
  name_colsy = colnames(an_y)[grepl("_name",colnames(an_y),ignore.case = T)]
  
  names_x = as.character(an_x[,name_colsx[1]])
  names_y = as.character(an_y[,name_colsy[1]])
  shared = intersect(names_x,names_y)
  shared = setdiff(shared,c(NA,""))
  
  mx = x$sample_data
  my = y$sample_data
  
  if(any(table(names_x)>1)){
    mx = apply(mx,2,function(x,y)tapply(x,INDEX=y,FUN=mean,na.rm=T),y=names_x)
  }
  else{
    rownames(mx) = as.character(names_x)
  }
  if(any(table(names_y)>1)){
    my = apply(my,2,function(x,y)tapply(x,INDEX=y,FUN=mean,na.rm=T),y=names_y)
  }
  else{
    rownames(my) = as.character(names_y)
  }
  
  bid_x = as.character(merged_dmaqc_data[colnames(mx),"bid"])
  bid_y = as.character(merged_dmaqc_data[colnames(my),"bid"])
  if(sum(!is.element(bid_y,set=bid_x))>0){
    stop("BIDs do not match between x and y")
  } # should be zero
  colnames(mx) = bid_x
  colnames(my) = bid_y
  if(length(shared)>samplesize){
    shared = sample(shared)[1:samplesize]
  }
  mx = mx[shared,bid_y]
  my = my[shared,]
  mx = as.matrix(mx);my = as.matrix(my)
  if(nrow(mx)<2){return(NA)}
  if(length(shared)<=100){
    corrs = diag(cor(t(mx),t(my)))
  }
  else{
    corrs = c()
    for(i in 1:nrow(mx)){
      # print(mean(mx[i,]))
      corrs[rownames(mx)[i]] = cor(mx[i,],my[i,])
    }
  }
  return(mean(corrs))
}
LS0tCnRpdGxlOiAiQklDIG1ldGFib2xvbWljcyBkYXRhIGFuYWx5c2lzIgpvdXRwdXQ6CiAgcGRmX2RvY3VtZW50OiAKICAgIG51bWJlcl9zZWN0aW9uczogdHJ1ZQogIGh0bWxfbm90ZWJvb2s6IGRlZmF1bHQKLS0tCgpJbiB0aGlzIGRvY3VtZW50IHdlIHByZXNlbnQgdGhlIGpvaW50IGFuYWx5c2lzIG9mIHRoZSBQQVNTMUEgbWV0YWJvbG9taWNzIGRhdGFzZXRzLgoKIyBMb2FkIGFsbCBkYXRhc2V0cwoKTG9hZCB0aGUgZGF0YSBmcm9tIHRoZSBjbG91ZCwgaW5jbHVkaW5nOiBwaGVub3R5cGljIGRhdGEsIG1ldGFib2xvbWljIGRhdGFzZXRzLCBhbmQgbWV0YWJvbG9taWNzIGRpY3Rpb25hcnkuCgpgYGB7cixyZXN1bHRzPSdoaWRlJyxtZXNzYWdlPUZBTFNFLHdhcm5pbmc9RkFMU0V9CnNvdXJjZSgifi9EZXNrdG9wL3JlcG9zL21vdHJwYWMtYmljLW5vcm0tcWMvdG9vbHMvc3VwZXJ2aXNlZF9ub3JtYWxpemF0aW9uX2Z1bmN0aW9ucy5SIikKc291cmNlKCJ+L0Rlc2t0b3AvcmVwb3MvbW90cnBhYy1iaWMtbm9ybS1xYy90b29scy91bnN1cGVydmlzZWRfbm9ybWFsaXphdGlvbl9mdW5jdGlvbnMuUiIpCnNvdXJjZSgifi9EZXNrdG9wL3JlcG9zL21vdHJwYWMtYmljLW5vcm0tcWMvdG9vbHMvZ2NwX2Z1bmN0aW9ucy5SIikKc291cmNlKCJ+L0Rlc2t0b3AvcmVwb3MvbW90cnBhYy1iaWMtbm9ybS1xYy90b29scy9hc3NvY2lhdGlvbl9hbmFseXNpc19tZXRob2RzLlIiKQpzb3VyY2UoIn4vRGVza3RvcC9yZXBvcy9tb3RycGFjLWJpYy1ub3JtLXFjL3Rvb2xzL2RhdGFfYXV4X2Z1bmN0aW9ucy5SIikKc291cmNlKCJ+L0Rlc2t0b3AvcmVwb3MvbW90cnBhYy90b29scy9wcmVkaWN0aW9uX21sX3Rvb2xzLlIiKQpsaWJyYXJ5KHJhbmRvbUZvcmVzdCkgIyBmb3IgY2xhc3NpZmljYXRpb24gdGVzdHMKCiMgTG9hZCB0aGUgZG1hcWMgZGF0YQptZXJnZWRfZG1hcWNfZGF0YSA9ICBsb2FkX2Zyb21fYnVja2V0KCJtZXJnZWRfZG1hcWNfZGF0YTIwMTktMTAtMTUuUkRhdGEiLAogICAgImdzOi8vYmljX2RhdGFfYW5hbHlzaXMvcGFzczFhL3BoZW5vX2RtYXFjLyIsRikKbWVyZ2VkX2RtYXFjX2RhdGEgPSBtZXJnZWRfZG1hcWNfZGF0YVtbMV1dCnJvd25hbWVzKG1lcmdlZF9kbWFxY19kYXRhKSA9IGFzLmNoYXJhY3RlcihtZXJnZWRfZG1hcWNfZGF0YSR2aWFsX2xhYmVsKQojIGRlZmluZSB0aGUgdGlzc3VlIHZhcmlhYmxlCm1lcmdlZF9kbWFxY19kYXRhJHRpc3N1ZSA9IG1lcmdlZF9kbWFxY19kYXRhJHNhbXBsZXR5cGVkZXNjcmlwdGlvbgojIGRlZmluZSB0aGUgdGltZSB0byBmcmVlemUgdmFyaWFibGUKbWVyZ2VkX2RtYXFjX2RhdGEkdGltZV90b19mcmVlemUgPSBtZXJnZWRfZG1hcWNfZGF0YSRjYWxjdWxhdGVkLnZhcmlhYmxlcy50aW1lX2RlYXRoX3RvX2NvbGxlY3RfbWluICsgCiAgbWVyZ2VkX2RtYXFjX2RhdGEkY2FsY3VsYXRlZC52YXJpYWJsZXMudGltZV9jb2xsZWN0X3RvX2ZyZWV6ZV9taW4KCiMgY29sIHRpbWUgdnMuIGNvbnRyb2wKIyBkZiA9IGRhdGEuZnJhbWUoCiMgICBiaWQgPSBtZXJnZWRfZG1hcWNfZGF0YSRiaWQsCiMgICBlZHRhX2NvbF90aW1lID0gbWVyZ2VkX2RtYXFjX2RhdGEkY2FsY3VsYXRlZC52YXJpYWJsZXMuZWR0YV9jb2xsX3RpbWUsCiMgICB0aW1lX3RvX2ZyZWV6ZSA9IG1lcmdlZF9kbWFxY19kYXRhJHRpbWVfdG9fZnJlZXplLAojICAgaXNfY29udHJvbCA9IG1lcmdlZF9kbWFxY19kYXRhJGFuaW1hbC5rZXkuaXNfY29udHJvbCwKIyAgIHRwID0gbWVyZ2VkX2RtYXFjX2RhdGEkYW5pbWFsLmtleS50aW1lcG9pbnQsCiMgICB0aXNzdWUgPSBtZXJnZWRfZG1hcWNfZGF0YSRzcGVjaW1lbi5wcm9jZXNzaW5nLnNhbXBsZXR5cGVkZXNjcmlwdGlvbgojICkKIyBkZiA9IHVuaXF1ZShkZikKIyBib3hwbG90KGVkdGFfY29sX3RpbWUvMzYwMCB+IGlzX2NvbnRyb2wsZGYpCiMgYm94cGxvdChlZHRhX2NvbF90aW1lLzM2MDAgLSB0cCB+IGlzX2NvbnRyb2wsZGYpCiMgd2lsY294LnRlc3QoZWR0YV9jb2xfdGltZS8zNjAwIH4gaXNfY29udHJvbCxkZikKCiMgYmxvb2QgZnJlZXplIHRpbWVzCmJsb29kX3NhbXBsZXMgPSAKICBtZXJnZWRfZG1hcWNfZGF0YSRzcGVjaW1lbi5wcm9jZXNzaW5nLnNhbXBsZXR5cGVkZXNjcmlwdGlvbiA9PQogICJFRFRBIFBsYXNtYSIKYmxvb2RfZnJlZXplX3RpbWUgPSAKICBhcy5kaWZmdGltZShtZXJnZWRfZG1hcWNfZGF0YSRzcGVjaW1lbi5wcm9jZXNzaW5nLnRfZnJlZXplLHVuaXRzID0gIm1pbnMiKSAtCiAgYXMuZGlmZnRpbWUobWVyZ2VkX2RtYXFjX2RhdGEkc3BlY2ltZW4ucHJvY2Vzc2luZy50X2VkdGFzcGluLHVuaXRzPSJtaW5zIikKYmxvb2RfZnJlZXplX3RpbWUgPSBhcy5udW1lcmljKGJsb29kX2ZyZWV6ZV90aW1lKQp0aW1lX3RvX2ZyZWV6ZSA9IG1lcmdlZF9kbWFxY19kYXRhJHRpbWVfdG9fZnJlZXplW2Jsb29kX3NhbXBsZXNdID0gCiAgYmxvb2RfZnJlZXplX3RpbWVbYmxvb2Rfc2FtcGxlc10KCiMgTG9hZCBvdXIgcGFyc2VkIG1ldGFib2xvbWljcyBkYXRhc2V0cwptZXRhYm9sb21pY3NfcGFyc2VkX2RhdGFzZXRzID0gbG9hZF9mcm9tX2J1Y2tldCgKICBmaWxlID0gIm1ldGFib2xvbWljc19wYXJzZWRfZGF0YXNldHNfcGFzczFhX2V4dGVybmFsMS5SRGF0YSIsCiAgYnVja2V0ID0gImdzOi8vYmljX2RhdGFfYW5hbHlzaXMvcGFzczFhL21ldGFib2xvbWljcy8iKVtbMV1dCgojICMgUmVhZCB0aGUgZGljdGlvbmFyeQojIGRpY3RfYnVja2V0ID0gCiMgICAiZ3M6Ly9tb3RycGFjLWV4dGVybmFsLXJlbGVhc2UxLXJlc3VsdHMvbWV0YWJvbG9taWNzX3RhcmdldGVkL21vdHJwYWNfbWV0YWJvbG9taWNzX2RhdGFfZGljdGlvbmFyeS12MS4xLjUudHh0IgojIGRpY3RfZG93bmxvYWQgPSBnZXRfc2luZ2xlX2ZpbGVfZnJvbV9idWNrZXRfdG9fbG9jYWxfZGlyKGRpY3RfYnVja2V0KQojIG1ldGFib2xvbWljc19kaWN0ID0gZnJlYWQoZGljdF9kb3dubG9hZFtbMV1dLGRhdGEudGFibGUgPSBGKQoKIyBQbG90IGN2IHZzIG1lYW5zCmxpYnJhcnkoZ3Bsb3RzKQpkID0gbWV0YWJvbG9taWNzX3BhcnNlZF9kYXRhc2V0c1tbIndoaXRlX2FkaXBvc2VfcG93ZGVyLG1ldGFiX3VfaGlsaWNwb3MsdW5uYW1lZCJdXQpkeCA9IGQkc2FtcGxlX2RhdGEKQ29WPC1mdW5jdGlvbih4KXtyZXR1cm4oc2QoeCxuYS5ybSA9IFQpL21lYW4oeCxuYS5ybT1UKSl9CmRtZWFucyA9IGFwcGx5KGR4LDEsbWVhbixuYS5ybT1UKQpDb1ZzID0gYXBwbHkoZHgsMSxDb1YpCmluZHMgPSAhaXMubmEoQ29WcykKZGYgPSBkYXRhLmZyYW1lKE1lYW5faW50ZW5zaXR5ID0gZG1lYW5zW2luZHNdLENvViA9IENvVnNbaW5kc10pCnBsb3QoQ29Wfk1lYW5faW50ZW5zaXR5LGRmLGNleD0wLjUscGNoPTIwKQpsaW5lcyhsb3dlc3MoQ29Wfk1lYW5faW50ZW5zaXR5LGRmKSxsdHk9Mixsd2Q9Mixjb2w9ImJsdWUiKQoKZHggPSBsb2coMStkJHNhbXBsZV9kYXRhLGJhc2U9MikKZG1lYW5zID0gYXBwbHkoZHgsMSxtZWFuLG5hLnJtPVQpCkNvVnMgPSBhcHBseShkeCwxLENvVikKaW5kcyA9ICFpcy5uYShDb1ZzKQpkZiA9IGRhdGEuZnJhbWUoTWVhbl9pbnRlbnNpdHkgPSBkbWVhbnNbaW5kc10sQ29WID0gQ29Wc1tpbmRzXSkKcGxvdChDb1Z+TWVhbl9pbnRlbnNpdHksZGYsY2V4PTAuNSxwY2g9MjApCmxpbmVzKGxvd2VzcyhDb1Z+TWVhbl9pbnRlbnNpdHksZGYpLGx0eT0yLGx3ZD0yLGNvbD0iYmx1ZSIpCgojIFBsb3QgbnVtYmVyIG9mIE5BcyB2cyBpbnRlbnNpdHkgbWVhbgpkeCA9IGxvZygxK2Qkc2FtcGxlX2RhdGEsYmFzZT0yKQpkbWVhbnMgPSBhcHBseShkeCwxLG1lYW4sbmEucm09VCkKbnVtX25hcyA9IHJvd1N1bXMoaXMubmEoZHgpKQpkZiA9IGRhdGEuZnJhbWUoTnVtX05BcyA9IG51bV9uYXNbaW5kc10sTWVhbl9pbnRlbnNpdHkgPSBkbWVhbnNbaW5kc10pCnBsb3QoTnVtX05Bc35NZWFuX2ludGVuc2l0eSxkZixjZXg9MC41LHBjaD0yMCkKbGluZXMobG93ZXNzKE51bV9OQXN+TWVhbl9pbnRlbnNpdHksZGYpLGx0eT0yLGx3ZD0yLGNvbD0iYmx1ZSIpCmNvcihkZiROdW1fTkFzLGRmJE1lYW5faW50ZW5zaXR5LG1ldGhvZD0ic3BlYXJtYW4iKQoKYGBgCgpEZWZpbmUgdGhlIHZhcmlhYmxlcyB0byBiZSBhZGp1c3RlZCBmb3I6CmBgYHtyfQpiaW9zcGVjX2NvbHMgPSBjKAogICJhY3V0ZS50ZXN0LmRpc3RhbmNlIiwKICAiY2FsY3VsYXRlZC52YXJpYWJsZXMudGltZV90b19mcmVlemUiLAogICMgImNhbGN1bGF0ZWQudmFyaWFibGVzLmVkdGFfY29sbF90aW1lIiwgIyBubyBuZWVkIC0gc2VlIGNvZGUgYWJvdmUgZm9yIGJsb29kCiAgImJpZCIgIyByZXF1aXJlZCBmb3IgbWF0Y2hpbmcgZGF0YXNldHMKICApCmRpZmZlcmVudGlhbF9hbmFseXNpc19jb2xzID0gYygKICAiYW5pbWFsLnJlZ2lzdHJhdGlvbi5zZXgiLAogICJhbmltYWwua2V5LnRpbWVwb2ludCIsCiAgImFuaW1hbC5rZXkuaXNfY29udHJvbCIKKQpwaXBlbGluZV9xY19jb2xzID0gYygic2FtcGxlX29yZGVyIikKYGBgCgojIERhdGEgZmlsdGVyaW5nIGFuZCBub3JtYWxpemF0aW9uCgpXZSBnbyBvdmVyIGVhY2ggZGF0YXNldCBhbmQgbWVyZ2UgdGhlIG5hbWVkIGFuZCB1bm5hbWVkIHN1YnBhcnRzIG9mIHRoZSB1bnRhcmdldGVkIGRhdGFzZXRzLiBGb3IgdGFyZ2V0ZWQgZGF0YSAtIHRoZSByZWxlYXNlIGJ1Y2tldHMgd2UgbWVyZ2UgZGF0YXNldHMgdGhhdCBhcmUgZnJvbSB0aGUgc2FtZSBzaXRlIGFuZCB0aXNzdWUuCgpUaGVuIHRoZSBwcmVwcm9jZXNzaW5nIG9mIHRoZSBuZXcgZGF0YXNldHMgaXMgYXMgZm9sbG93czogKDEpIGxvZyB0aGUgZGF0YSAodmFsdWVzIGFyZSByYXcgaW50ZW5zaXRpZXMpLCAoMikgcmVtb3ZlIHJvd3Mgd2l0aCAkPjIwXCUkIG1pc3NpbmcgdmFsdWVzLCBhbmQgKDMpIGltcHV0ZSBtaXNzaW5nIHZhbHVlcy4KCkltcG9ydGFudDogZGF0YXNldHMgdGhhdCBkbyBub3QgaGF2ZSB1bmlxdWUgbWV0YWJvbGl0ZSBuYW1lcyBvciBzYW1wbGUgaWRzIHByb2JhYmx5IGhhZCBlcnJvcnMgd2hpbGUgcGFyc2luZyBhbmQgYXJlIHRoZXJlZm9yZSBpZ25vcmVkLiBBbHNvLCB1bnRhcmdldGVkIHdpdGggZmFpbGVkIG1lcmduaW5nIG9mIHRoZSB1bm5hbWVkIGFuZCBuYW1lZCBzdWJzZXRzIChlLmcuLCBkdWUgdG8gZGlmZmVyZW50IHNhbXBsZSBpZHMpIGFyZSBpZ25vcmVkIGFzIHdlbGwuCgpGaW5hbGx5IHdlIGFkZCBhbiBhZGRpdGlvbmFsIHZlcnNpb24gZm9yIGVhY2ggZGF0YXNldCBieSBkaXJlY3RseSByZWdyZXNzaW5nIG91dCB0aGUgc2FtcGxlIG9yZGVyIGNvbXBvbmVudC4gU2VlIGh0dHBzOi8vd3d3Lm5jYmkubmxtLm5paC5nb3YvcG1jL2FydGljbGVzL1BNQzQ3NTc2MDMvIGZvciBtb3JlIGRldGFpbHMgKHRyZW5kIGNvcnJlY3Rpb24gaXMgYWxzbyBjYWxsZWQgYmFja2dyb3VuZCBjb3JyZWN0aW9uLCBhbmQgdGhpcyBpcyBkb25lIGZvciBlYWNoIGJhdGNoIHNlcGFyYXRlbHkpLgoKYGBge3Isb3V0LmhlaWdodD0nNTAlJyxvdXQud2lkdGg9JzUwJScsbWVzc2FnZT1ULHdhcm5pbmc9RkFMU0V9Cm1ldGFib2xvbWljc19wcm9jZXNzZWRfZGF0YXNldHMgPSBjKCkKIyBzZXQgYSBiaW5hcnkgdmVjdG9yIGluZGljYXRpbmcgd2l0aCBlbnRyaWVzIHRvIHNraXAKbWV0YWJvbG9taWNzX3Jhd19kYXRhc2V0c19za2lwID0gcmVwKEYsbGVuZ3RoKG1ldGFib2xvbWljc19wYXJzZWRfZGF0YXNldHMpKQpuYW1lcyhtZXRhYm9sb21pY3NfcmF3X2RhdGFzZXRzX3NraXApID0gbmFtZXMobWV0YWJvbG9taWNzX3BhcnNlZF9kYXRhc2V0cykKdW50YXJnZXRlZF9tZXJnZV9lcnJvcnMgPSBsaXN0KCkKZm9yKGN1cnJuYW1lIGluIG5hbWVzKG1ldGFib2xvbWljc19wYXJzZWRfZGF0YXNldHMpKXsKICAKICBpZihtZXRhYm9sb21pY3NfcmF3X2RhdGFzZXRzX3NraXBbY3Vycm5hbWVdKXtuZXh0fQogIAogIGFyciA9IHN0cnNwbGl0KGN1cnJuYW1lLHNwbGl0PSIsIilbWzFdXQogIGFyciA9IGFyclstbGVuZ3RoKGFycildCiAgbmFtZTEgPSBwYXN0ZShwYXN0ZShhcnIsY29sbGFwc2U9IiwiKSwibmFtZWQiLHNlcD0iLCIpCiAgbmFtZTIgPSBwYXN0ZShwYXN0ZShhcnIsY29sbGFwc2U9IiwiKSwidW5uYW1lZCIsc2VwPSIsIikKICAKICAjIElmIGRhdGFzZXQgaXMgdW50YXJnZXRlZCwgbWVyZ2UgbmFtZWQgYW5kIHVubmFtZWQsIHVwZGF0ZSB0aGUgZGF0YXNldAogICMgY3Vycm5hbWUuIFNraXAgdGhlIGRhdGFzZXQgaWYgdGhlIG1lcmdlIGZhaWxzLCBidXQgc3RvcmUgYW5kIHByaW50IHRoZSBlcnJyb3IuCiAgIyBJZiB0aGUgZGF0YXNldCBpcyB0YXJnZXRlZCAtIGdvIG92ZXIgYWxsIGRhdGFzZXRzIHdpdGggdGhlIHNhbWUgdmlhbCBsYWJlbHMgYW5kIAogICMgbWVyZ2UgaWYgcG9zc2libGUuIAogIGlmKG5hbWUyICVpbiUgbmFtZXMobWV0YWJvbG9taWNzX3BhcnNlZF9kYXRhc2V0cykpewogICAgZDEgPSBtZXRhYm9sb21pY3NfcGFyc2VkX2RhdGFzZXRzW1tuYW1lMV1dCiAgICBkMiA9IG1ldGFib2xvbWljc19wYXJzZWRfZGF0YXNldHNbW25hbWUyXV0KICAgIG5ld2QgPSBOVUxMCiAgICB0cnlDYXRjaCh7CiAgICAgIG5ld2QgPSBtZXJnZV9uYW1lZF9hbmRfdW5uYW1lZF9tZXRhYm9sb21pY3NfZGF0YXNldHMoZDEsZDIsc3RyaWN0ID0gRikKICAgICAgfSwgZXJyb3IgPSBmdW5jdGlvbihlKSB7fSkgIyBjYW4gYWRkIGVycm9yIGhhbmRsaW5nIGhlcmUKICAgIGN1cnJuYW1lID0gcGFzdGUocGFzdGUoYXJyLGNvbGxhcHNlPSIsIiksInVudGFyZ2V0ZWQiLHNlcD0iLCIpCiAgICBtZXRhYm9sb21pY3NfcmF3X2RhdGFzZXRzX3NraXBbbmFtZTFdID0gVAogICAgbWV0YWJvbG9taWNzX3Jhd19kYXRhc2V0c19za2lwW25hbWUyXSA9IFQKICAgIGlmKGlzLm51bGwobmV3ZCkpewogICAgICBwcmludChwYXN0ZSgiIyBTa2lwcGluZyBkYXRhc2V0IiwKICAgICAgICAgICAgICAgICAgY3Vycm5hbWUsCiAgICAgICAgICAgICAgICAgICJiZWNhdXNlIG1lcmdpbmcgdGhlIG5hbWVkIGFuZCB1bm5hbWVkIHN1YnNldHMgZmFpbGVkIikpCiAgICAgIG5leHQKICAgIH0KICB9CiAgZWxzZXsKICAgIG1ldGFib2xvbWljc19yYXdfZGF0YXNldHNfc2tpcFtjdXJybmFtZV0gPSBUCiAgICBuZXdkID0gbWV0YWJvbG9taWNzX3BhcnNlZF9kYXRhc2V0c1tbY3Vycm5hbWVdXQogICAgY3Vycl92aWFscyA9IGNvbG5hbWVzKG5ld2Qkc2FtcGxlX2RhdGEpCiAgICBjdXJyX3NpdGUgPSB0b2xvd2VyKHVuaXF1ZShuZXdkJHNhbXBsZV9tZXRhJHNpdGUpKQogICAgZm9yKGFub3RoZXJfZGF0YXNldCBpbiBuYW1lcyhtZXRhYm9sb21pY3NfcmF3X2RhdGFzZXRzX3NraXApKXsKICAgICAgaWYobWV0YWJvbG9taWNzX3Jhd19kYXRhc2V0c19za2lwW2Fub3RoZXJfZGF0YXNldF0pe25leHR9CiAgICAgIGFub3RoZXJfZCA9IG1ldGFib2xvbWljc19wYXJzZWRfZGF0YXNldHNbW2Fub3RoZXJfZGF0YXNldF1dCiAgICAgIGFub3RoZXJfdmlhbHMgPSBjb2xuYW1lcyhhbm90aGVyX2Qkc2FtcGxlX2RhdGEpCiAgICAgIGFub3RoZXJfZF9zaXRlID0gdG9sb3dlcih1bmlxdWUoYW5vdGhlcl9kJHNhbXBsZV9tZXRhJHNpdGUpKQogICAgICBhbm90aGVyX3NoYXJlZF92aWFscyA9IGludGVyc2VjdChhbm90aGVyX3ZpYWxzLGN1cnJfdmlhbHMpCiAgICAgIGlmKGN1cnJfc2l0ZSA9PSBhbm90aGVyX2Rfc2l0ZSAmJgogICAgICAgIGxlbmd0aChhbm90aGVyX3NoYXJlZF92aWFscyk9PWxlbmd0aChhbm90aGVyX3ZpYWxzKSAmJgogICAgICAgIGxlbmd0aChhbm90aGVyX3NoYXJlZF92aWFscyk9PWxlbmd0aChjdXJyX3ZpYWxzKQogICAgICApewogICAgICAgICAgIyBleGFtaW5lIHRoZSBtZXRhZGF0YQogICAgICAgICAgY3Vycl9tZXRhID0gbmV3ZCRzYW1wbGVfbWV0YQogICAgICAgICAgYW5vdGhlcl9tZXRhID0gYW5vdGhlcl9kJHNhbXBsZV9tZXRhCiAgICAgICAgICAjIGF0IHRoaXMgcG9pbnQgd2Uga25vdyB0aGF0IGJvdGggZGF0YXNldHMgaGF2ZSB0aGUKICAgICAgICAgICMgc2FtZSBzYW1wbGVzLCBidXQgbm90IG5lY2Vzc2FyaWx5IHRoZSBzYW1lIGNvbnRyb2xzCiAgICAgICAgICAjIHRoZXNlIGhhcyBiZWVuIHBhcnRpYWxseSBzdWJtaXR0ZWQgYW5kIHdlIHRoZXJlZm9yZSBuZWVkIHRvCiAgICAgICAgICAjIGludGVyc2VjdCBoZXJlCiAgICAgICAgICBzaGFyZWRfbWV0YV9zYW1wbGVzID0gaW50ZXJzZWN0KHJvd25hbWVzKGN1cnJfbWV0YSkscm93bmFtZXMoYW5vdGhlcl9tZXRhKSkKICAgICAgICAgICMgbWFrZSBzdXJlIHRoYXQgdGhlIHZpYWxzLCB0eXBlcywgYW5kIG9yZGVyIGFyZSB0ZSBzYW1lCiAgICAgICAgICBpZihsZW5ndGgoc2hhcmVkX21ldGFfc2FtcGxlcykgPCAxIHx8CiAgICAgICAgICAgICFhbGwoY3Vycl9tZXRhW3NoYXJlZF9tZXRhX3NhbXBsZXMsMTozXT09YW5vdGhlcl9tZXRhW3NoYXJlZF9tZXRhX3NhbXBsZXMsMTozXSkpewogICAgICAgICAgICBuZXh0CiAgICAgICAgICB9CiAgICAgICAgICB0cnkoewogICAgICAgICAgICBuZXdkID0gbWVyZ2VfbmFtZWRfYW5kX3VubmFtZWRfbWV0YWJvbG9taWNzX2RhdGFzZXRzKAogICAgICAgICAgICAgIG5ld2QsYW5vdGhlcl9kLHN0cmljdCA9IEYpICMgbm8gc3RyaWN0IG1lcmdlCiAgICAgICAgICAgIG1ldGFib2xvbWljc19yYXdfZGF0YXNldHNfc2tpcFthbm90aGVyX2RhdGFzZXRdID0gVAogICAgICAgICAgfSkKICAgICAgfQogICAgfQogICAgCiAgICAjIHVwZGF0ZSB0aGUgbmFtZSBvZiB0aGUgZGF0YXNldAogICAgIyB3ZSBrZWVwIHRyYWNrIG9mIHRoZSBudW1iZXIgb2YgZGF0YXNldHMgZnJvbSB0aGUgZ2l2ZW4gc2l0ZSwgYXMgc29tZQogICAgIyBtZXJnZXMgbWF5IGZhaWwKICAgIGN1cnJuYW1lID0gY3Vycm5hbWUgPSBwYXN0ZShhcnJbMV0sY3Vycl9zaXRlLHNlcD0iLCIpCiAgICBudW1fY3Vycm5hbWUgPSBzdW0oZ3JlcGwoY3Vycm5hbWUsbmFtZXMobWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0cykpKQogICAgY3Vycm5hbWUgPSBwYXN0ZShjdXJybmFtZSxudW1fY3Vycm5hbWUrMSxzZXA9IiwiKQogIH0KICAKICBwcmludChwYXN0ZSgiIyMjIyBBbmFseXppbmcgZGF0YSBmcm9tOiIsY3Vycm5hbWUpKQogIGN1cnJfZGF0YSAgPSBuZXdkJHNhbXBsZV9kYXRhCiAgY3Vycl9kYXRhMiA9IHJlY2FzdF9udW1lcmljX2RhdGFfZnJhbWUoY3Vycl9kYXRhKQogIGN1cnJfZGF0YV9tYXQgPSBhcy5tYXRyaXgoY3Vycl9kYXRhMikKICBwcmludChwYXN0ZSgibm8gZXJyb3JzIGluIHBhcnNpbmcgbnVtZXJpYyB2YWx1ZXM6IixhbGwoY3Vycl9kYXRhPT1jdXJyX2RhdGFfbWF0LG5hLnJtPVQpKSkKCiAgIyBmbG9vciB0aGUgZGF0YSBhdCAwCiAgY3Vycl9kYXRhX21hdFtjdXJyX2RhdGFfbWF0PDBdID0gMAogICMgS2VlcCBhIGxvZ2dlZCB2ZXJzaW9uIG9mIHRoZSBkYXRhCiAgY3Vycl9kYXRhX2xvZyA9IGxvZyhjdXJyX2RhdGFfbWF0KzEsYmFzZT0yKQogIAogICMgb3JnYW5pemUgdGhlIG1ldGFkYXRhCiAgY3Vycl9tZXRhID0gbWVyZ2VkX2RtYXFjX2RhdGFbY29sbmFtZXMoY3Vycl9kYXRhKSwKICAgICAgICB1bmlvbihiaW9zcGVjX2NvbHMsZGlmZmVyZW50aWFsX2FuYWx5c2lzX2NvbHMpXQogICMgcmVtb3ZlIG1ldGFkYXRhIHZhcmlhYmxlcyB3aXRoIHRvbyBtYW55IE5BcwogIG5hX2NvdW50cyA9IGFwcGx5KGlzLm5hKGN1cnJfbWV0YSksMixzdW0pCiAgY3Vycl9tZXRhID0gY3Vycl9tZXRhWyxuYV9jb3VudHMvbnJvdyhjdXJyX21ldGEpIDwgMC4xXQogIAogICMgTG9vayBhdCB0aGUgc2FtcGxlIG9yZGVyCiAgY3Vycl9vcmRlciA9IG5ld2Qkc2FtcGxlX21ldGFbcm93bmFtZXMoY3Vycl9tZXRhKSwic2FtcGxlX29yZGVyIl0KCiAgIyBvcmdhbml6ZSB0aGUgZGF0YXNldAogIGN1cnJfZGF0YSA9IGN1cnJfZGF0YVsscm93bmFtZXMoY3Vycl9tZXRhKV0KICBjdXJyX2RhdGFfbG9nID0gY3Vycl9kYXRhX2xvZ1sscm93bmFtZXMoY3Vycl9tZXRhKV0KICAjIHJlbW92ZSB6ZXJvIHZhcmlhbmNlIHJvd3Mgb3Igcm93cyB3aXRoIG1hbnkgTkFzCiAgcm93c190b19yZW0gPSAhKGFwcGx5KGN1cnJfZGF0YSwxLHNkLG5hLnJtPVQpPjApCiAgcGVyY2VudF9uYSA9IHJvd1N1bXMoaXMubmEoY3Vycl9kYXRhKSkgLyBuY29sKGN1cnJfZGF0YSkKICBwcmludChwYXN0ZSgibnVtYmVyIG9mIE5BIGNlbGxzOiIsc3VtKGlzLm5hKGN1cnJfZGF0YSkpKSkKICByb3dzX3RvX3JlbSA9IHJvd3NfdG9fcmVtIHwgcGVyY2VudF9uYSA+IDAuMgogICMgcmVtb3ZlIHJvd3Mgd2l0aCBubyBhbm5vdGF0aW9uCiAgcm93c190b19yZW0gPSByb3dzX3RvX3JlbSB8IG5ld2QkZGF0YV9yYXdfcm93bmFtZXMgPT0gIiIKICByb3dzX3RvX3JlbSA9IHJvd3NfdG9fcmVtIHwgbmV3ZCRkYXRhX3Jhd19yb3duYW1lcyA9PSAiLSIKICByb3dzX3RvX3JlbSA9IHJvd3NfdG9fcmVtIHwgbmV3ZCRkYXRhX3Jhd19yb3duYW1lcyA9PSAiXyIKICByb3dzX3RvX3JlbSA9IHJvd3NfdG9fcmVtIHwgaXMubmEobmV3ZCRkYXRhX3Jhd19yb3duYW1lcykKICAjIHJlbW92ZSByb3dzIHdpdGggZHVwbGljYXRlZCByZXByZXNlbnRhdGlvbgogIHJvd19kdXBsaWNhdGlvbnMgPSBuYW1lcyh3aGljaCh0YWJsZShuZXdkJGRhdGFfcmF3X3Jvd25hbWVzKT4xKSkKICBpZihhbnkobmV3ZCRkYXRhX3Jhd19yb3duYW1lc1shcm93c190b19yZW1dICVpbiUgcm93X2R1cGxpY2F0aW9ucykpewogICAgcm93c190b19yZW0gPSByb3dzX3RvX3JlbSB8IG5ld2QkZGF0YV9yYXdfcm93bmFtZXMgJWluJSByb3dfZHVwbGljYXRpb25zCiAgICBwcmludCgiIyBXQVJOSU5HOiBkYXRhc2V0IGhhcyBkdXBsaWNhdGVkIHJvdyBuYW1lcyB3aG9zZSBkYXRhIGFyZSBub3cgaWdub3JlZCIpCiAgICBwcmludChwYXN0ZShyb3dfZHVwbGljYXRpb25zLGNvbGxhcHNlPSIsIikpCiAgfQogIGN1cnJfZGF0YSA9IGN1cnJfZGF0YVshcm93c190b19yZW0sXQogIGN1cnJfZGF0YV9sb2cgPSBjdXJyX2RhdGFfbG9nWyFyb3dzX3RvX3JlbSxdCiAgIyBpbXB1dGUgLSB1c2UgY2FwdHVyZS5vdXRwdXQgdG8gYXZvaWQgcHJpbnRzCiAgY2FwdHVyZS5vdXRwdXQoCiAgICB7Y3Vycl9kYXRhX2ltcCA9IGltcHV0ZTo6aW1wdXRlLmtubihhcy5tYXRyaXgoY3Vycl9kYXRhX2xvZykpJGRhdGF9CiAgICAsZmlsZT1OVUxMKQogIGN1cnJfZGF0YV9ybmFtZXMgPSBuZXdkJGRhdGFfcmF3X3Jvd25hbWVzWyFyb3dzX3RvX3JlbV0KICByb3duYW1lcyhjdXJyX2RhdGFfaW1wKSA9IGN1cnJfZGF0YV9ybmFtZXMKICAKICAjIFJlZ3Jlc3Mgb3V0IHRoZSBzYW1wbGUgb3JkZXIKICAjIGxtX3JlZ3Jlc3NfbWF0cml4IHdvcmtzIG9uIGNvbHVtbnMKICBjdXJyX2RhdGFfaW1wMiA9IHQobG1fcmVncmVzc19vdXRfbWF0cml4KHQoY3Vycl9kYXRhX2ltcCksY3Vycl9vcmRlcikpCiAgCiAgIyB1cGRhdGUgdGhlIGRhdGEgb2JqZWN0CiAgbWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0c1tbY3Vycm5hbWVdXSA9IG5ld2QKICBtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzW1tjdXJybmFtZV1dJHNhbXBsZV9kYXRhID0gY3Vycl9kYXRhX2ltcAogIG1ldGFib2xvbWljc19wcm9jZXNzZWRfZGF0YXNldHNbW2N1cnJuYW1lXV0kc2FtcGxlX2RhdGFfbm9sb2dfbm9pbXAgPSBjdXJyX2RhdGEKICBtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzW1tjdXJybmFtZV1dJGRhdGFfcmF3X3Jvd25hbWVzID0gY3Vycl9kYXRhX3JuYW1lcwogIG1ldGFib2xvbWljc19wcm9jZXNzZWRfZGF0YXNldHNbW2N1cnJuYW1lXV0kc2FtcGxlX21ldGFfcGFyc2VkID0gY3Vycl9tZXRhCiAgbWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0c1tbY3Vycm5hbWVdXSRzYW1wbGVfZGF0YV9vcmRlcl9hZGogPSBjdXJyX2RhdGFfaW1wMgp9CgpzYXZlX3RvX2J1Y2tldChtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzLAogICAgICAgICAgICAgICBmaWxlPSJtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzMTAyODIwMTkuUkRhdGEiLAogICAgICAgICAgICAgICBidWNrZXQgPSAiZ3M6Ly9iaWNfZGF0YV9hbmFseXNpcy9wYXNzMWEvbWV0YWJvbG9taWNzLyIpCgpgYGAKCgpBbHRlcm5hdGl2ZWx5IGxvYWQgdGhlIHJlc3VsdCBmcm9tIHRoZSBidWNrZXQgdG8gc2F2ZSB0aW1lOgpgYGB7cn0KbWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0cyA9IGxvYWRfZnJvbV9idWNrZXQoCiAgZmlsZT0ibWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0czEwMjgyMDE5LlJEYXRhIiwKICBidWNrZXQgPSAiZ3M6Ly9iaWNfZGF0YV9hbmFseXNpcy9wYXNzMWEvbWV0YWJvbG9taWNzLyIKKVtbMV1dCmBgYAoKIyBFeGFtaW5lIHRoZSB1bmxvZ2dlZCBkYXRhCgpVbnRhcmdldGVkIGRhdGEgYXJlIHR5cGljYWxseSBsb2dnZWQgYW5kIGFuYWx5emVkIHVzaW5nIGxpbmVhciBtb2RlbHMuIE9uIHRoZSBvdGhlciBoYW5kLCBjb25jZW50cmF0aW9uIGRhdGEgYXJlIGFuYWx5emVkIHdpdGggdGhlIHNhbWUgdHlwZSBvZiBtb2RlbHMgYnV0IHVzaW5nIHRoZSBvcmlnaW5hbCBkYXRhLiBUaGlzIHJhaXNlcyBhIHByb2JsZW0gaWYgd2Ugd2lzaCB0byBjb21wYXJlIGV4YWN0IHN0YXRpc3RpY3MgZnJvbSB0aGVzZSBkYXRhLiBJbiB0aGlzIHNlY3Rpb24gd2UgcGVyZm9ybSByZXNpZHVhbCBhbmFseXNpcyBmb3Igc2luZ2xlIG1ldGFib2xpdGVzLiBPdXIgZ29hbCBpcyB0byBpZGVudGlmeSBpZiBjb25jZW50cmF0aW9uIGRhdGEgYmVoYXZlcyAibm9ybWFsbHkiIHdoZW4gbm90IGxvZ2dlZC4gVGhlIGFuYWx5c2lzIGJlbG93IGV4YW1pbmVzIHRoZSByZXNpZHVhbHMgb2YgdGhlIGRhdGEgYW5kIHVzZXMgcXEtcGxvdHMgdG8gZGV0ZXJtaW5lIGlmIGEgbG9nIHRyYW5zZm9ybWF0aW9uIGlzIGluZGVlZCByZXF1aXJlZC4KCmBgYHtyLG91dC5oZWlnaHQ9JzUwJScsb3V0LndpZHRoPSc1MCUnLG1lc3NhZ2U9VCx3YXJuaW5nPUZBTFNFfQppc19ub3JtYWxfdGVzdDwtZnVuY3Rpb24odixzYW1wPTEwMDAwKXsKICByZXR1cm4oa3MudGVzdCh2LCJwbm9ybSIsbWVhbih2LG5hLnJtPVQpLHNkKHYsbmEucm0gPSBUKSkkcC52YWx1ZSkKfQojIGdvIG92ZXIgdGhlIG5hbWVkIGRhdGFzZXRzLCBnZXQgYSBsb2dnZWQgYW5kIGFuIHVubG9nZ2VkIHZlcnNpb24gb2YKIyB0aGUgZGF0YSwgdXNlIHRoZXNlIGFzIGlucHV0cyBmb3IgdGhlIHJlZ3Jlc3Npb24KcmVzaWR1YWxfYW5hbHlzaXNfcmVzdWx0cyA9IGxpc3QoKQpmb3Iobm4xIGluIG5hbWVzKG1ldGFib2xvbWljc19wcm9jZXNzZWRfZGF0YXNldHMpKXsKICBpZihncmVwbCgidW50YXJnZXRlZCIsbm4xKSl7bmV4dH0KICB4X2xvZyA9IG1ldGFib2xvbWljc19wcm9jZXNzZWRfZGF0YXNldHNbW25uMV1dJHNhbXBsZV9kYXRhCiAgeF91bmxvZyA9IDJeeF9sb2cKICAKICAjIHRha2UgdGhlIGNvdmFyaWF0ZXMsIGlnbm9yZSBkaXN0YW5jZXMKICB4X21ldGEgPSB1bmlxdWUobWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0c1tbbm4xXV0kc2FtcGxlX21ldGFfcGFyc2VkKQogIGN1cnJfY292cyA9IHhfbWV0YVssaW50ZXJzZWN0KGNvbG5hbWVzKHhfbWV0YSksYmlvc3BlY19jb2xzWzJdKV0KICBjdXJyX2NvdnMgPSBkYXRhLmZyYW1lKGN1cnJfY292cywKICAgICAgICAgICBzZXg9eF9tZXRhJGFuaW1hbC5yZWdpc3RyYXRpb24uc2V4KQogIAogICMgZ2V0IHRoZSBsbSBvYmplY3RzCiAgY3Vycl9tb2RlbHMgPSBsaXN0KCkKICBmb3IodHAgaW4gdW5pcXVlKHhfbWV0YSRhbmltYWwua2V5LnRpbWVwb2ludCkpewogICAgICByZXNfbG9nID0gYXBwbHkoCiAgICAgICAgeF9sb2csMSwKICAgICAgICBwYXNzMWFfc2ltcGxlX2RpZmZlcmVudGlhbF9hYnVuZGFuY2UsCiAgICAgICAgdHBzID0geF9tZXRhJGFuaW1hbC5rZXkudGltZXBvaW50LHRwPXRwLAogICAgICAgIGlzX2NvbnRyb2wgPSB4X21ldGEkYW5pbWFsLmtleS5pc19jb250cm9sLAogICAgICAgIGNvdnMgPSBjdXJyX2NvdnMscmV0dXJuX21vZGVsPVQKICAgICAgKQogICAgICByZXNfdW5sb2cgPSBhcHBseSgKICAgICAgICB4X3VubG9nLDEsCiAgICAgICAgcGFzczFhX3NpbXBsZV9kaWZmZXJlbnRpYWxfYWJ1bmRhbmNlLAogICAgICAgIHRwcyA9IHhfbWV0YSRhbmltYWwua2V5LnRpbWVwb2ludCx0cD10cCwKICAgICAgICBpc19jb250cm9sID0geF9tZXRhJGFuaW1hbC5rZXkuaXNfY29udHJvbCwKICAgICAgICBjb3ZzID0gY3Vycl9jb3ZzLHJldHVybl9tb2RlbD1UCiAgICAgICkKICAgICAgaXNfbm9ybSA9IGNiaW5kKAogICAgICAgIHNhcHBseShyZXNfbG9nLGZ1bmN0aW9uKHgpaXNfbm9ybWFsX3Rlc3QocmVzaWR1YWxzKHgpKSksCiAgICAgICAgc2FwcGx5KHJlc191bmxvZyxmdW5jdGlvbih4KWlzX25vcm1hbF90ZXN0KHJlc2lkdWFscyh4KSkpCiAgICAgICkKICAgICAgY29sbmFtZXMoaXNfbm9ybSkgPSBjKCJsb2ciLCJub3QgbG9nIikKICAgICAgY3Vycl9tb2RlbHNbW2FzLmNoYXJhY3Rlcih0cCldXSA9IGlzX25vcm0KICB9CiAgcmVzaWR1YWxfYW5hbHlzaXNfcmVzdWx0c1tbbm4xXV0gPSBjdXJyX21vZGVscwp9Cgpsb2dfdnNfdW5sb2dfc3VtbV9tYXQgPSBzYXBwbHkocmVzaWR1YWxfYW5hbHlzaXNfcmVzdWx0cywKICAgIGZ1bmN0aW9uKHgpc2FwcGx5KHgsCiAgICAgICAgZnVuY3Rpb24oeSkKICAgICAgICAgIHdpbGNveC50ZXN0KHlbLDFdLHlbLDJdLHBhaXJlZCA9IFQsYWx0ZXJuYXRpdmUgPSAiZyIpJHAudmFsdWUpKQoKbnVtX25vbm5vcm1hbF9sb2cgPSBzYXBwbHkocmVzaWR1YWxfYW5hbHlzaXNfcmVzdWx0cywKICAgIGZ1bmN0aW9uKHgpc2FwcGx5KHgsCiAgICAgICAgZnVuY3Rpb24oeSlzdW0oeVssMV08MC4wNSkpKQpudW1fbm9ubm9ybWFsX2xvZyA9IAogIG51bV9ub25ub3JtYWxfbG9nWyxvcmRlcihjb2xuYW1lcyhudW1fbm9ubm9ybWFsX2xvZykpXQpudW1fbm9ubm9ybWFsX3VubG9nID0gc2FwcGx5KHJlc2lkdWFsX2FuYWx5c2lzX3Jlc3VsdHMsCiAgICBmdW5jdGlvbih4KXNhcHBseSh4LAogICAgICAgIGZ1bmN0aW9uKHkpc3VtKHlbLDJdPDAuMDUpKSkKbnVtX25vbm5vcm1hbF91bmxvZyA9IAogIG51bV9ub25ub3JtYWxfdW5sb2dbLG9yZGVyKGNvbG5hbWVzKG51bV9ub25ub3JtYWxfdW5sb2cpKV0KCmxpYnJhcnkoY29ycnBsb3QpCnBhcihtYXIgPSBjKDUsNSw1LDEwKSkKY29ycnBsb3QodCgtbG9nMTAobG9nX3ZzX3VubG9nX3N1bW1fbWF0KSksaXMuY29yciA9IEYpCmNvcnJwbG90KHQobnVtX25vbm5vcm1hbF9sb2cpLSB0KG51bV9ub25ub3JtYWxfdW5sb2cpLAogICAgICAgICBpcy5jb3JyID0gRix0bC5jZXggPSAwLjcpCgoKYGBgCgoKIyBBbmFseXNpcyBvZiBzcGVjaWZpYyBtZXRhYm9saXRlcwoKQ29tcGFyZSBvdmVybGFwcywgZWZmZWN0IHNpemVzLCBhbmQgY29ycmVsYXRpb25zIHdpdGhpbiB0aXNzdWVzLiBDb21wYXJlIHRhcmdldGVkLXVudGFyZ2V0ZWQgcGFpcnMgb25seS4KCmBgYHtyLG91dC5oZWlnaHQ9JzUwJScsb3V0LndpZHRoPSc1MCUnLG1lc3NhZ2U9VCx3YXJuaW5nPUZBTFNFfQoKIyBoZWxwZXIgZnVuY3Rpb24gdG8gdHJhbnNmb3JtIGEgbWV0YWJvbG9taWNzIG1hdHJpeAojIHRvIHRoYXQgb2YgaXRzIG1vdHJwYWMgY29tcG91bmQgaWRzCmV4dHJhY3RfbWV0YWJfZGF0YV9mcm9tX3Jvd19hbm5vdDwtZnVuY3Rpb24oeCxyb3dfYW5ub3RfeCl7CiAgIyBnZXQgdGhlIGNvbG91bW4gdGhhdCBoYXMgdGhlIHJvdyBuYW1lcwogIGludF9zaXplcyA9IGFwcGx5KHJvd19hbm5vdF94LDIsZnVuY3Rpb24oeCx5KWxlbmd0aChpbnRlcnNlY3QoeCx5KSkseT1yb3duYW1lcyh4KSkKICBpbmQgPSB3aGljaChpbnRfc2l6ZXM9PW1heChpbnRfc2l6ZXMsbmEucm0gPSBUKSlbMV0KICByb3dfYW5ub3RfeCA9IHJvd19hbm5vdF94W2lzLmVsZW1lbnQocm93X2Fubm90X3hbLGluZF0sc2V0PXJvd25hbWVzKHgpKSxdCiAgcm93bmFtZXMocm93X2Fubm90X3gpID0gcm93X2Fubm90X3hbLGluZF0KICBzaGFyZWQgPSBpbnRlcnNlY3Qocm93bmFtZXMocm93X2Fubm90X3gpLHJvd25hbWVzKHgpKQogIHggPSB4W3NoYXJlZCxdCiAgcm93X2Fubm90X3ggPSByb3dfYW5ub3RfeFtzaGFyZWQsXQogIHJvd25hbWVzKHgpID0gcm93X2Fubm90X3gkbW90cnBhY19jb21wX25hbWUKICByZXR1cm4oeCkKfQoKc2luZ2xlX21ldGFib2xpdGVfY29ycnMgPSBsaXN0KCkKc2luZ2xlX21ldGFib2xpdGVfZGUgPSBjKCkKbmFtZWQyY292ZXJlZF9zaGFyZWRfbWV0YWJvbGl0ZXMgPSBsaXN0KCkKZm9yKG5uMSBpbiBuYW1lcyhtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzKSl7CiAgbm4xX3Rpc3N1ZSA9IHN0cnNwbGl0KG5uMSxzcGxpdD0iLCIpW1sxXV1bMV0KICBubjFfdGlzc3VlID0gZ3N1YigiX3Bvd2RlciIsIiIsbm4xX3Rpc3N1ZSkKICBpZihncmVwbCgidW50YXJnZXRlZCIsbm4xKSl7bmV4dH0KICBzaW5nbGVfbWV0YWJvbGl0ZV9jb3Jyc1tbbm4xXV0gPSBsaXN0KCkKICBuYW1lZDJjb3ZlcmVkX3NoYXJlZF9tZXRhYm9saXRlc1tbbm4xXV0gPSBOVUxMCiAgZm9yKG5uMiBpbiBuYW1lcyhtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzKSl7CiAgICBpZihubjIgPT0gbm4xKXtuZXh0fQogICAgaWYoIWdyZXBsKCJ1bnRhcmdldGVkIixubjIpKXtuZXh0fQogICAgbm4yX3Rpc3N1ZSA9IHN0cnNwbGl0KG5uMixzcGxpdD0iLCIpW1sxXV1bMV0KICAgIG5uMl90aXNzdWUgPSBnc3ViKCJfcG93ZGVyIiwiIixubjJfdGlzc3VlKQogICAgbm4yX2RhdGFzZXQgPSBzdHJzcGxpdChubjIsc3BsaXQ9IiwiKVtbMV1dWzJdCiAgICBpZihubjFfdGlzc3VlIT1ubjJfdGlzc3VlKXtuZXh0fQogICAgcHJpbnQobm4yKQogICAgIyBnZXQgdGhlIG51bWVyaWMgZGF0YXNldAogICAgeCA9IG1ldGFib2xvbWljc19wcm9jZXNzZWRfZGF0YXNldHNbW25uMV1dJHNhbXBsZV9kYXRhCiAgICAjIHggPSBsb2cyKHgrMSkKICAgIHkgPSBtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzW1tubjJdXSRzYW1wbGVfZGF0YQogICAgIyBwcmludChwYXN0ZSgiZGF0YXNldDE6IixubjEpKQogICAgIyBwcmludChyYW5nZSh4KSkKICAgICMgcHJpbnQocGFzdGUoImRhdGFzZXQyOiIsbm4yKSkKICAgICMgcHJpbnQocmFuZ2UoeSkpCiAgICByb3dfYW5ub3RfeCA9IG1ldGFib2xvbWljc19wcm9jZXNzZWRfZGF0YXNldHNbW25uMV1dJHJvd19hbm5vdAogICAgcm93X2Fubm90X3kgPSBtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzW1tubjJdXSRyb3dfYW5ub3QKICAgICMgdHJhbnNmb3JtIG1ldGFib2xpdGUgbmFtZXMgdG8gdGhlIG1vdHJwYWMgY29tcCBuYW1lCiAgICB4ID0gZXh0cmFjdF9tZXRhYl9kYXRhX2Zyb21fcm93X2Fubm90KHgscm93X2Fubm90X3gpCiAgICB5ID0gZXh0cmFjdF9tZXRhYl9kYXRhX2Zyb21fcm93X2Fubm90KHkscm93X2Fubm90X3kpCiAgICAjIGFsaWduIHRoZSBzYW1wbGUgc2V0cwogICAgYmlkX3kgPSBtZXJnZWRfZG1hcWNfZGF0YVtjb2xuYW1lcyh5KSwiYmlkIl0KICAgIGJpZF94ID0gbWVyZ2VkX2RtYXFjX2RhdGFbY29sbmFtZXMoeCksImJpZCJdICAgIAogICAgIyBzdGVwIDE6IG1lcmdlIHNhbXBsZXMgZnJvbSB0aGUgc2FtZSBCSUQKICAgIGlmKGxlbmd0aCh1bmlxdWUoYmlkX3gpKSE9bGVuZ3RoKGJpZF94KSl7CiAgICAgIHggPSBhZ2dyZWdhdGVfcmVwZWF0ZWRfc2FtcGxlcyh4LGJpZF94KQogICAgfQogICAgZWxzZXsKICAgICAgY29sbmFtZXMoeCkgPSBiaWRfeAogICAgfQogICAgaWYobGVuZ3RoKHVuaXF1ZShiaWRfeSkpIT1sZW5ndGgoYmlkX3kpKXsKICAgICAgeSA9IGFnZ3JlZ2F0ZV9yZXBlYXRlZF9zYW1wbGVzKHksYmlkX3kpCiAgICB9ZWxzZXsKICAgICAgY29sbmFtZXMoeSkgPSBiaWRfeQogICAgfQogICAgIyBzdGVwIDI6IHVzZSB0aGUgc2hhcmVkIGJpbyBpZHMKICAgIHNoYXJlZF9iaWRzID0gYXMuY2hhcmFjdGVyKGludGVyc2VjdChjb2xuYW1lcyh5KSxjb2xuYW1lcyh4KSkpCiAgICB4ID0gYXMubWF0cml4KHhbLHNoYXJlZF9iaWRzXSkKICAgIHkgPSBhcy5tYXRyaXgoeVssc2hhcmVkX2JpZHNdKQogICAgIyBBdCB0aGlzIHBvaW50IHggYW5kIHkgYXJlIG92ZXIgdGhlIHNhbWUgQklEcywgbm93IHdlIGFkZCB0aGUgbWV0YWRhdGEKICAgIHlfbWV0YSA9IHVuaXF1ZShtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzW1tubjFdXSRzYW1wbGVfbWV0YV9wYXJzZWQpCiAgICByb3duYW1lcyh5X21ldGEpID0geV9tZXRhJGJpZAogICAgeV9tZXRhID0geV9tZXRhW3NoYXJlZF9iaWRzLF0KICAgIAogICAgIyBnZXQgdGhlIHNoYXJlZCBtYXRlYm9saXRlcwogICAgc2hhcmVkX21ldGFib2xpdGVzID0gaW50ZXJzZWN0KHJvd25hbWVzKHgpLHJvd25hbWVzKHkpKQogICAgc2hhcmVkX21ldGFib2xpdGVzID0gbmEub21pdChzaGFyZWRfbWV0YWJvbGl0ZXMpCiAgICBpZihsZW5ndGgoc2hhcmVkX21ldGFib2xpdGVzKT09MCl7bmV4dH0KICAgIG5hbWVkMmNvdmVyZWRfc2hhcmVkX21ldGFib2xpdGVzW1tubjFdXSA9IHVuaW9uKAogICAgICBuYW1lZDJjb3ZlcmVkX3NoYXJlZF9tZXRhYm9saXRlc1tbbm4xXV0sCiAgICAgIHNoYXJlZF9tZXRhYm9saXRlcwogICAgKQogICAgCiAgICAjIEdldCBzdGF0aXN0aWNzCiAgICBpZihsZW5ndGgoc2hhcmVkX21ldGFib2xpdGVzKT4xKXsKICAgICAgICAgIGNvcnJzID1jb3IodCh4W3NoYXJlZF9tZXRhYm9saXRlcyxdKSwKICAgICAgICAgICAgICAgIHQoeVtzaGFyZWRfbWV0YWJvbGl0ZXMsXSksbWV0aG9kID0gInNwZWFybWFuIikKICAgIH0KICAgIGVsc2V7CiAgICAgICAgICBjb3JycyA9IGNvcih4W3NoYXJlZF9tZXRhYm9saXRlcyxdLAogICAgICAgICAgICAgICAgeVtzaGFyZWRfbWV0YWJvbGl0ZXMsXSxtZXRob2QgPSAic3BlYXJtYW4iKQogICAgfQogICAgCiAgICAjIHRha2UgdGhlIGNvdmFyaWF0ZXMsIGlnbm9yZSBkaXN0YW5jZXMKICAgIGN1cnJfY292X2NvbHMgPSBpbnRlcnNlY3QoY29sbmFtZXMoeV9tZXRhKSxiaW9zcGVjX2NvbHNbMl0pCiAgICBjdXJyX2NvdnMgPSBkYXRhLmZyYW1lKHlfbWV0YVssY3Vycl9jb3ZfY29sc10pCiAgICBuYW1lcyhjdXJyX2NvdnMpID0gY3Vycl9jb3ZfY29scwogICAgIyAjIHNjYWxlCiAgICAjIGZvcihjb3ZfbmFtZSBpbiBuYW1lcyhjdXJyX2NvdnMpKXsKICAgICMgICBjdXJyX2NvdnNbW2Nvdl9uYW1lXV0gPSBzY2FsZShjdXJyX2NvdnNbW2Nvdl9uYW1lXV0sY2VudGVyID0gVCxzY2FsZSA9IFQpCiAgICAjIH0KICAgIGN1cnJfY292cyRzZXggPSB5X21ldGEkYW5pbWFsLnJlZ2lzdHJhdGlvbi5zZXgKICAgIAogICAgIyBkaWZmZXJlbnRpYWwgYW5hbHlzaXMKICAgIGZvcih0cCBpbiB1bmlxdWUoeV9tZXRhJGFuaW1hbC5rZXkudGltZXBvaW50KSl7CiAgICAgIHJlc3ggPSB0KGFwcGx5KAogICAgICAgIG1hdHJpeCh4W3NoYXJlZF9tZXRhYm9saXRlcyxdLG5yb3c9bGVuZ3RoKHNoYXJlZF9tZXRhYm9saXRlcykpLDEsCiAgICAgICAgcGFzczFhX3NpbXBsZV9kaWZmZXJlbnRpYWxfYWJ1bmRhbmNlLAogICAgICAgIHRwcyA9IHlfbWV0YSRhbmltYWwua2V5LnRpbWVwb2ludCx0cD10cCwKICAgICAgICBpc19jb250cm9sID0geV9tZXRhJGFuaW1hbC5rZXkuaXNfY29udHJvbCwKICAgICAgICBjb3ZzID0gY3Vycl9jb3ZzLHJldHVybl9tb2RlbD1GCiAgICAgICkpCiAgICAgIHJlc3kgPSB0KGFwcGx5KAogICAgICAgIG1hdHJpeCh5W3NoYXJlZF9tZXRhYm9saXRlcyxdLG5yb3c9bGVuZ3RoKHNoYXJlZF9tZXRhYm9saXRlcykpLDEsCiAgICAgICAgcGFzczFhX3NpbXBsZV9kaWZmZXJlbnRpYWxfYWJ1bmRhbmNlLAogICAgICAgIHRwcyA9IHlfbWV0YSRhbmltYWwua2V5LnRpbWVwb2ludCx0cD10cCwKICAgICAgICBpc19jb250cm9sID0geV9tZXRhJGFuaW1hbC5rZXkuaXNfY29udHJvbCwKICAgICAgICBjb3ZzID0gY3Vycl9jb3ZzLHJldHVybl9tb2RlbD1GCiAgICAgICkpCiAgICAgICMgQWRkIGRhdGFzZXQgaW5mb3JtYXRpb24sIHRpbWUgcG9pbnQsIHRpc3N1ZQogICAgICBhZGRlZF9jb2x1bW5zID0gbWF0cml4KGNiaW5kKAogICAgICAgIHJlcChubjFfZGF0YXNldCxsZW5ndGgoc2hhcmVkX21ldGFib2xpdGVzKSksCiAgICAgICAgcmVwKG5uMl9kYXRhc2V0LGxlbmd0aChzaGFyZWRfbWV0YWJvbGl0ZXMpKSwKICAgICAgICBzaGFyZWRfbWV0YWJvbGl0ZXMsCiAgICAgICAgcmVwKHRwLGxlbmd0aChzaGFyZWRfbWV0YWJvbGl0ZXMpKSwKICAgICAgICByZXAobm4xX3Rpc3N1ZSxsZW5ndGgoc2hhcmVkX21ldGFib2xpdGVzKSkKICAgICAgKSxucm93PWxlbmd0aChzaGFyZWRfbWV0YWJvbGl0ZXMpKQogICAgICByZXN4ID0gY2JpbmQocmVzeCxyZXAoVCxucm93KHJlc3gpKSkKICAgICAgY29sbmFtZXMocmVzeClbbmNvbChyZXN4KV0gPSAiaXNfdGFyZ2V0ZWQiCiAgICAgIHJlc3kgPSBjYmluZChyZXN5LHJlcChGLG5yb3cocmVzeSkpKQogICAgICBjb2xuYW1lcyhyZXN5KVtuY29sKHJlc3kpXSA9ICJpc190YXJnZXRlZCIKICAgICAgaWYobnJvdyhyZXN4KT4xKXsKICAgICAgICByZXN4ID0gY2JpbmQoYWRkZWRfY29sdW1uc1ssLTJdLHJlc3gpCiAgICAgICAgcmVzeSA9IGNiaW5kKGFkZGVkX2NvbHVtbnNbLC0xXSxyZXN5KQogICAgICB9CiAgICAgIGVsc2V7CiAgICAgICAgcmVzeCA9IGMoYWRkZWRfY29sdW1uc1ssLTJdLHJlc3gpCiAgICAgICAgcmVzeSA9IGMoYWRkZWRfY29sdW1uc1ssLTFdLHJlc3kpCiAgICAgIH0KICAgICAgc2luZ2xlX21ldGFib2xpdGVfZGUgPSByYmluZChzaW5nbGVfbWV0YWJvbGl0ZV9kZSxyZXN4KQogICAgICBzaW5nbGVfbWV0YWJvbGl0ZV9kZSA9IHJiaW5kKHNpbmdsZV9tZXRhYm9saXRlX2RlLHJlc3kpCiAgICB9CiAgICAKICAgIHNpbmdsZV9tZXRhYm9saXRlX2NvcnJzW1tubjFdXVtbbm4yXV0gPSBjb3JycwogIH0KfQpzaW5nbGVfbWV0YWJvbGl0ZV9kZSA9IGRhdGEuZnJhbWUoc2luZ2xlX21ldGFib2xpdGVfZGUpCm5hbWVzKHNpbmdsZV9tZXRhYm9saXRlX2RlKSA9IGMoCiAgImRhdGFzZXQiLCJtZXRhYm9saXRlIiwidHAiLCJ0aXNzdWUiLAogICJFc3QiLCJTdGQiLCJUc3RhdCIsIlB2YWx1ZSIsImlzX3RhcmdldGVkIikKZm9yKGNvbCBpbiBuYW1lcyhzaW5nbGVfbWV0YWJvbGl0ZV9kZSlbLWMoMTo0KV0pewogIHNpbmdsZV9tZXRhYm9saXRlX2RlW1tjb2xdXSA9IGFzLm51bWVyaWMoCiAgICBhcy5jaGFyYWN0ZXIoc2luZ2xlX21ldGFib2xpdGVfZGVbW2NvbF1dKSkKfQpmb3IoY29sIGluIG5hbWVzKHNpbmdsZV9tZXRhYm9saXRlX2RlKVsxOjRdKXsKICBzaW5nbGVfbWV0YWJvbGl0ZV9kZVtbY29sXV0gPSAKICAgIGFzLmNoYXJhY3RlcihzaW5nbGVfbWV0YWJvbGl0ZV9kZVtbY29sXV0pCn0KCmBgYAoKV2UgbmV4dCB0cmFuc2Zvcm0gdGhlIGRhdGEgYWJvdmUgaW50byB0YWJsZXMgdGhhdCBjb250YWluIGRhdGEgZm9yIGVhY2ggY29tYmluYXRpb24gb2YgbWV0YWJvbGl0ZSwgdGltZSBwb2ludCwgYW5kIHRpc3N1ZS4gVGhlc2UgYXJlIHRoZW4gdXNlZCBmb3IgdHdvIG1ldGEtYW5hbHlzZXM6ICgxKSBhIHNpbXBsZSByYW5kb20gZWZmZWN0cyBhbmFseXNpcywgYW5kICgyKSByYW5kb20gZWZmZWN0cyB3aXRoIGEgYmluYXJ5IGNvdmFyaWF0ZSBpbmRpY2F0aW5nIGlmIGEgZGF0YXNldCBpcyB0YXJnZXRlZCBvciB1bnRhcmdldGVkLgoKYGBge3Isb3V0LmhlaWdodD0nNTAlJyxvdXQud2lkdGg9JzUwJScsbWVzc2FnZT1GLHdhcm5pbmc9RkFMU0V9Cgpyb3duYW1lcyhzaW5nbGVfbWV0YWJvbGl0ZV9kZSkgPSBOVUxMCmZvcihubiBpbiBuYW1lcyhzaW5nbGVfbWV0YWJvbGl0ZV9kZSkpewogIG5kaWcgPSA1CiAgaWYoZ3JlcGwoInB2YWwiLG5uLGlnbm9yZS5jYXNlID0gVCkpewogICAgbmRpZyA9IDEwCiAgfQogIGlmKGlzLm51bWVyaWMoc2luZ2xlX21ldGFib2xpdGVfZGVbW25uXV0pKXsKICAgIHNpbmdsZV9tZXRhYm9saXRlX2RlW1tubl1dID0gcm91bmQoc2luZ2xlX21ldGFib2xpdGVfZGVbW25uXV0sZGlnaXRzID0gbmRpZykKICB9Cn0Kc2luZ2xlX21ldGFib2xpdGVfZGUgPSB1bmlxdWUoc2luZ2xlX21ldGFib2xpdGVfZGUpCgpsaWJyYXJ5KG1ldGFmb3IpCm1ldGFfYW5hbHlzaXNfc3RhdHMgPSBsaXN0KCkKZm9yKHRpc3N1ZSBpbiB1bmlxdWUoc2luZ2xlX21ldGFib2xpdGVfZGUkdGlzc3VlKSl7CiAgZm9yKHRwIGluIHVuaXF1ZShzaW5nbGVfbWV0YWJvbGl0ZV9kZSR0cCkpewogICAgY3Vycl9zdWJzZXQgPSBzaW5nbGVfbWV0YWJvbGl0ZV9kZVsKICAgICAgc2luZ2xlX21ldGFib2xpdGVfZGUkdGlzc3VlPT10aXNzdWUgJgogICAgICAgIHNpbmdsZV9tZXRhYm9saXRlX2RlJHRwPT10cCxdCiAgICBmb3IobWV0YWJvbGl0ZSBpbiB1bmlxdWUoY3Vycl9zdWJzZXQkbWV0YWJvbGl0ZSkpewogICAgICBjdXJyX21ldF9kYXRhID0gY3Vycl9zdWJzZXRbCiAgICAgICAgY3Vycl9zdWJzZXQkbWV0YWJvbGl0ZT09bWV0YWJvbGl0ZSxdCiAgICAgIGN1cnJfbWV0X2RhdGEkdmFyID0gY3Vycl9tZXRfZGF0YSRTdGReMgogICAgICByZV9tb2RlbDEgPSBOVUxMO3JlX21vZGVsMj1OVUxMO3JlX21vZGVsX3RhciA9IE5VTEwKICAgICAgdHJ5KHtyZV9tb2RlbDEgPSBybWEudW5pKGN1cnJfbWV0X2RhdGEkRXN0LGN1cnJfbWV0X2RhdGEkdmFyKX0pCiAgICAgIHRyeSh7cmVfbW9kZWwyID0gcm1hLm12KGN1cnJfbWV0X2RhdGEkRXN0LGN1cnJfbWV0X2RhdGEkdmFyLAogICAgICAgICAgICAgICAgICAgICAgICAgbW9kcz1jdXJyX21ldF9kYXRhJGlzX3RhcmdldGVkKX0pCiAgICAgIHRyeSh7cmVfbW9kZWxfdGFyID0gcm1hLnVuaSgKICAgICAgICBjdXJyX21ldF9kYXRhW2N1cnJfbWV0X2RhdGEkaXNfdGFyZ2V0ZWQ9PTEsIkVzdCJdLAogICAgICAgIGN1cnJfbWV0X2RhdGFbY3Vycl9tZXRfZGF0YSRpc190YXJnZXRlZD09MSwidmFyIl0KICAgICAgKX0pCiAgICAgIG1ldGFfYW5hbHlzaXNfc3RhdHNbW3Bhc3RlKG1ldGFib2xpdGUsdGlzc3VlLHRwLHNlcD0iLCIpXV0gPSAKICAgICAgICBsaXN0KGN1cnJfbWV0X2RhdGE9Y3Vycl9tZXRfZGF0YSxyZV9tb2RlbDE9cmVfbW9kZWwxLAogICAgICAgICAgICByZV9tb2RlbDIgPSByZV9tb2RlbDIscmVfbW9kZWxfdGFyPXJlX21vZGVsX3RhcikKICAgIH0KICB9Cn0KCgpgYGAKCldlIG5vdyBzaG93IHNvbWUgcGxvdHMgdG8gc3VtbWFyaXplIHRoZSBjb21wYXJpc29uLiBXZSBwbG90IHRoZSBvdmVyYWxsIGF2ZXJhZ2UgY29ycmVsYXRpb24gYmV0d2VlbiB0aGUgcGxhdGZvcm1zICh3aXRoaW4gdGlzc3VlcykuIEhvd2V2ZXIsIGdvaW5nIGludG8gYSBzaW5nbGUgbWV0YWJvbGl0ZSBjb21wYXJpc29uIGluIGRldGFpbCAoYWdhaW4sIHdpdGhpbiB0aXNzdWVzKSwgZm9yIGVhY2ggbWV0YWJvbGl0ZSwgaW4gZWFjaCB0aW1lIHBvaW50IHdlIHBlcmZvcm0gYSBtZXRhLWFuYWx5c2lzIG9mIHRoZSBlZmZlY3RzIGFuZCBleGFtaW5lIHRoZSBzaWduaWZpY2FuY2UgYW5kIGhldGVyb2dlbmVpdHkuCgpgYGB7cixvdXQuaGVpZ2h0PSc1MCUnLG91dC53aWR0aD0nNTAlJyxtZXNzYWdlPUYsd2FybmluZz1GQUxTRX0KIyBEbyB3ZSBoYXZlIG5pY2UgY292ZXJhZ2VzIG9mIHRoZSBuYW1lZCBkYXRhc2V0cz8KbGlicmFyeShnZ3Bsb3QyKQpkYXRhc2V0Mm51bV9tZXRhYm9saXRlcyA9IHNhcHBseShtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnVuY3Rpb24oeClucm93KHgkc2FtcGxlX2RhdGEpKQpuYW1lZF9kYXRhc2V0X2NvdmVyYWdlID0gc2FwcGx5KG5hbWVkMmNvdmVyZWRfc2hhcmVkX21ldGFib2xpdGVzLGxlbmd0aCkKbmFtZWRfZGF0YXNldF9jb3ZlcmFnZSA9IGRhdGEuZnJhbWUoCiAgbmFtZSA9IG5hbWVzKG5hbWVkX2RhdGFzZXRfY292ZXJhZ2UpLAogIHBlcmNlbnRhZ2UgPSBuYW1lZF9kYXRhc2V0X2NvdmVyYWdlIC8KICBkYXRhc2V0Mm51bV9tZXRhYm9saXRlc1tuYW1lcyhuYW1lZF9kYXRhc2V0X2NvdmVyYWdlKV0sCiAgY291bnQgPSBuYW1lZF9kYXRhc2V0X2NvdmVyYWdlLAogIHRvdGFsID0gZGF0YXNldDJudW1fbWV0YWJvbGl0ZXNbbmFtZXMobmFtZWRfZGF0YXNldF9jb3ZlcmFnZSldCikKIyBhZGQgZGF0YXNldHMgd2l0aCBubyBjb3ZlcmFnZQphbGxfdGFyZ2V0ZWRfZGF0YXNldHMgPSBuYW1lcyhtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzKQphbGxfdGFyZ2V0ZWRfZGF0YXNldHMgPSBhbGxfdGFyZ2V0ZWRfZGF0YXNldHNbIWdyZXBsKCJ1bnRhcmciLGFsbF90YXJnZXRlZF9kYXRhc2V0cyldCnplcm9fY292ZXJhZ2VfZGF0YXNldHMgPSBzZXRkaWZmKGFsbF90YXJnZXRlZF9kYXRhc2V0cywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbmFtZWRfZGF0YXNldF9jb3ZlcmFnZSRuYW1lKQp6ZXJvX2NvdmVyYWdlX2RhdGFzZXRzID0gZGF0YS5mcmFtZSgKICBuYW1lID0gemVyb19jb3ZlcmFnZV9kYXRhc2V0cywKICBwZXJjZW50YWdlID0gcmVwKDAsbGVuZ3RoKHplcm9fY292ZXJhZ2VfZGF0YXNldHMpKSwKICBjb3VudCA9IHJlcCgwLGxlbmd0aCh6ZXJvX2NvdmVyYWdlX2RhdGFzZXRzKSksCiAgdG90YWwgPSBkYXRhc2V0Mm51bV9tZXRhYm9saXRlc1t6ZXJvX2NvdmVyYWdlX2RhdGFzZXRzXQopCm5hbWVkX2RhdGFzZXRfY292ZXJhZ2UgPSByYmluZChuYW1lZF9kYXRhc2V0X2NvdmVyYWdlLAogICAgICAgICAgICAgICAgICAgICAgICAgICB6ZXJvX2NvdmVyYWdlX2RhdGFzZXRzKQpuYW1lZF9kYXRhc2V0X2NvdmVyYWdlID0gCiAgbmFtZWRfZGF0YXNldF9jb3ZlcmFnZVtvcmRlcihhcy5jaGFyYWN0ZXIobmFtZWRfZGF0YXNldF9jb3ZlcmFnZSRuYW1lKSksXQpwcmludChnZ3Bsb3QobmFtZWRfZGF0YXNldF9jb3ZlcmFnZSwgYWVzKHg9bmFtZSwgeT1wZXJjZW50YWdlKSkgKyAKICBnZW9tX2JhcihzdGF0ID0gImlkZW50aXR5Iix3aWR0aD0wLjIpICsgY29vcmRfZmxpcCgpICsKICBnZW9tX3RleHQoZGF0YT1uYW1lZF9kYXRhc2V0X2NvdmVyYWdlLCAKICAgICAgICAgICAgYWVzKG5hbWUsIHBlcmNlbnRhZ2UrMC4wNSwgbGFiZWw9Y291bnQpLCAKICAgICAgICAgICAgcG9zaXRpb24gPSBwb3NpdGlvbl9kb2RnZSh3aWR0aD0wLjkpLAogICAgICAgICAgICBzaXplPTQpICsgCiAgZ2d0aXRsZSgiVGFyZ2V0ZWQgZGF0YXNldDogY292ZXJhZ2UgYnkgdW50YXJnZXRlZCIpKQoKCiMgTmV4dCBleGFtaW5lIHRoZSBTcGVhcm1hbiBjb3JyZWxhdGlvbnMgYmV0d2VlbiBwbGF0Zm9ybXMKZXh0cmFjdF9kaWFnX3ZzX25vbl9kaWFnPC1mdW5jdGlvbihjb3JycyxmdW5jPW1lYW4sLi4uKXsKICBpZihsZW5ndGgoY29ycnMpPT0xKXsKICAgIHJldHVybihjKHNhbWU9ZnVuYyhjb3JycywuLi4pLG90aGVyPU5BKSkKICB9CiAgc2FtZSA9IGZ1bmMoZGlhZyhjb3JycyksLi4uKQogIG90aGVyID0gZnVuYygKICAgIGMoY29ycnNbbG93ZXIudHJpKGNvcnJzLGRpYWcgPSBGKV0pLC4uLikKICByZXR1cm4oYyhzYW1lPXNhbWUsb3RoZXI9b3RoZXIpKQp9CiMgdGFyX2RhdGFzZXQydGlzc3VlID0gc2FwcGx5KG5hbWVzKHNpbmdsZV9tZXRhYm9saXRlX2NvcnJzKSwKIyAgICAgICBmdW5jdGlvbih4KXN0cnNwbGl0KHgsc3BsaXQ9IiwiKVtbMV1dWzFdKQojIHRhcl9kYXRhc2V0MnRpc3N1ZSA9IGdzdWIoIl9wb3dkZXIiLCIiLHRhcl9kYXRhc2V0MnRpc3N1ZSkKZm9yKHRhcl9kYXRhc2V0IGluIG5hbWVzKHNpbmdsZV9tZXRhYm9saXRlX2NvcnJzKSl7CiAgbCA9IHNpbmdsZV9tZXRhYm9saXRlX2NvcnJzW1t0YXJfZGF0YXNldF1dCiAgaWYobGVuZ3RoKGwpPT0wKXtuZXh0fQogIGNvcnJfaW5mbyA9IGFzLmRhdGEuZnJhbWUodChzYXBwbHkobCwgZXh0cmFjdF9kaWFnX3ZzX25vbl9kaWFnKSkpCiAgY29ycl9zZCA9IGFzLmRhdGEuZnJhbWUodChzYXBwbHkobCwgZXh0cmFjdF9kaWFnX3ZzX25vbl9kaWFnLGZ1bmM9c2QpKSkKICByb3duYW1lcyhjb3JyX2luZm8pID0gc2FwcGx5KHJvd25hbWVzKGNvcnJfaW5mbyksCiAgICAgIGZ1bmN0aW9uKHgpc3Ryc3BsaXQoeCxzcGxpdD0iLCIpW1sxXV1bMl0pCiAgcm93bmFtZXMoY29ycl9pbmZvKSA9IGdzdWIoIm1ldGFiX3VfIiwiIixyb3duYW1lcyhjb3JyX2luZm8pKQogIHJvd25hbWVzKGNvcnJfc2QpID0gcm93bmFtZXMoY29ycl9pbmZvKQogIGNvcnJfaW5mbyRkYXRhc2V0ID0gcm93bmFtZXMoY29ycl9pbmZvKQogIGNvcnJfc2QkZGF0YXNldCA9IGNvcnJfaW5mbyRkYXRhc2V0CiAgY29ycl9pbmZvID0gbWVsdChjb3JyX2luZm8pCiAgY29ycl9zZCA9IG1lbHQoY29ycl9zZCkKICBjb3JyX2luZm8kc2QgPSBjb3JyX3NkJHZhbHVlCiAgcHJpbnQoCiAgICBnZ3Bsb3QoY29ycl9pbmZvLCBhZXMoeD1kYXRhc2V0LCB5PXZhbHVlLCBmaWxsPXZhcmlhYmxlKSkgKwogICAgICBnZW9tX2Jhcihwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSgpLCBzdGF0PSJpZGVudGl0eSIsIGNvbG91cj0nYmxhY2snKSArCiAgICAgIGdlb21fZXJyb3JiYXIoYWVzKHltaW49dmFsdWUtc2QsIHltYXg9dmFsdWUrc2QpLG5hLnJtPVQsIAogICAgICAgICAgICAgICAgICAgd2lkdGg9LjIscG9zaXRpb249cG9zaXRpb25fZG9kZ2UoLjkpKSArCiAgICBnZ3RpdGxlKHRhcl9kYXRhc2V0KSArIHhsYWIoIlVudGFyZ2V0ZWQgZGF0YXNldCIpICsgeWxhYigiU3BlYXJtYW4iKSArCiAgICAgIGxhYnMoZmlsbCA9ICJQYWlyIHR5cGUiKSArIAogICAgICB0aGVtZShsZWdlbmQucG9zaXRpb249InRvcCIsbGVnZW5kLmRpcmVjdGlvbiA9ICJob3Jpem9udGFsIikKICApCn0KCiMgTG9vayBhdCB0aGUgbWV0YS1hbmFseXNpcyByZXN1bHRzIAojIEV4dHJhY3Qgc29tZSB1c2VmdWwgc3RhdGlzdGljcyBwZXIgYW5hbHlzaXMKcHZhbHNfdGFyID0gc2FwcGx5KG1ldGFfYW5hbHlzaXNfc3RhdHMsZnVuY3Rpb24oeCl4JHJlX21vZGVsMSRwdmFsKQoKSTJfc2NvcmVzID0gc2FwcGx5KG1ldGFfYW5hbHlzaXNfc3RhdHMsZnVuY3Rpb24oeCl4JHJlX21vZGVsMSRJMikKSTJfc2NvcmVzID0gdW5saXN0KEkyX3Njb3Jlc1tzYXBwbHkoSTJfc2NvcmVzLGxlbmd0aCk+MF0pCmhpc3QoSTJfc2NvcmVzKQpiZXRhcyA9IHNhcHBseShtZXRhX2FuYWx5c2lzX3N0YXRzLGZ1bmN0aW9uKHgpeCRyZV9tb2RlbDEkYmV0YVsxLDFdKQpiZXRhcyA9IHVubGlzdChiZXRhc1tuYW1lcyhJMl9zY29yZXMpXSkKcHZhbHMgPSBzYXBwbHkobWV0YV9hbmFseXNpc19zdGF0cyxmdW5jdGlvbih4KXgkcmVfbW9kZWwxJHB2YWwpCnB2YWxzID0gdW5saXN0KHB2YWxzW25hbWVzKEkyX3Njb3JlcyldKQp0YXJnZXRlZF9kaWZmX3AgPSAKICBzYXBwbHkobWV0YV9hbmFseXNpc19zdGF0cyxmdW5jdGlvbih4KXgkcmVfbW9kZWwyJHB2YWxbMl0pCnRhcmdldGVkX2RpZmZfcCA9IHVubGlzdCh0YXJnZXRlZF9kaWZmX3BbbmFtZXMoSTJfc2NvcmVzKV0pCnBsb3QoYmV0YXMsSTJfc2NvcmVzKQpwbG90KC1sb2cxMChwdmFscyksSTJfc2NvcmVzKQpwbG90KC1sb2cxMCh0YXJnZXRlZF9kaWZmX3ApLEkyX3Njb3JlcykKcGxvdCgtbG9nMTAodGFyZ2V0ZWRfZGlmZl9wKSwtbG9nMTAocHZhbHMpKQojIHBsb3QgZXhhbXBsZXMKYWdyZWVfZXhhbXBsZSA9IG5hbWVzKHNhbXBsZSh3aGljaChwdmFsczwgMC4wMDEgJiBJMl9zY29yZXMgPCAyMCkpWzFdKQpmb3Jlc3QobWV0YV9hbmFseXNpc19zdGF0c1tbYWdyZWVfZXhhbXBsZV1dJHJlX21vZGVsMSwKICAgICAgIHNsYWIgPSBtZXRhX2FuYWx5c2lzX3N0YXRzW1thZ3JlZV9leGFtcGxlXV1bWzFdXVssMV0sCiAgICAgICBtYWluID0gYWdyZWVfZXhhbXBsZSx4bGFiID0gIkxvZyBmYyIsCiAgICAgICBjb2wgPSAiYmx1ZSIsY2V4ID0gMS4xKQpkaXNhZ3JlZV9leGFtcGxlID0gbmFtZXMoc2FtcGxlKHdoaWNoKHRhcmdldGVkX2RpZmZfcDwgMC4wMDEpKSlbMV0KZm9yZXN0KG1ldGFfYW5hbHlzaXNfc3RhdHNbW2Rpc2FncmVlX2V4YW1wbGVdXSRyZV9tb2RlbDEsCiAgICAgICBzbGFiID0gbWV0YV9hbmFseXNpc19zdGF0c1tbZGlzYWdyZWVfZXhhbXBsZV1dW1sxXV1bLDFdLAogICAgICAgbWFpbiA9IGRpc2FncmVlX2V4YW1wbGUseGxhYiA9ICJMb2cgZmMiLAogICAgICAgY29sID0gImJsdWUiLGNleCA9IDEuMSkKCgpgYGAKCgojIERhdGFzZXQgcGFpcndpc2UgY29tcGFyaXNvbiBhcyBhIHByZWRpY3Rpb24gdGFzawoKVXNlIDEwLWZvbGQgY3Jvc3MgdmFsaWRhdGlvbiBmb3IgYW5hbHlzaXMgd2l0aGluIHRpc3N1ZXMuCgpgYGB7cixmaWcuYWxpZ249ImNlbnRlciIsb3V0LmhlaWdodD0nNjAlJyxvdXQud2lkdGg9JzYwJScsZXZhbD1GQUxTRX0KbmZvbGRzID0gMTAKcHJlZGljdGlvbl9hbmFseXNpc19yZXN1bHRzID0gbGlzdCgpCmZvcihubjEgaW4gbmFtZXMobWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0cykpewogIG5uMV90aXNzdWUgPSBzdHJzcGxpdChubjEsc3BsaXQ9IiwiKVtbMV1dWzFdCiAgbm4xX3Rpc3N1ZSA9IGdzdWIoIl9wb3dkZXIiLCIiLG5uMV90aXNzdWUpCiAgZm9yKG5uMiBpbiBuYW1lcyhtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzKSl7CiAgICBubjJfdGlzc3VlID0gc3Ryc3BsaXQobm4yLHNwbGl0PSIsIilbWzFdXVsxXQogICAgbm4yX3Rpc3N1ZSA9IGdzdWIoIl9wb3dkZXIiLCIiLG5uMl90aXNzdWUpCiAgICBpZihubjFfdGlzc3VlIT1ubjJfdGlzc3VlKXtuZXh0fQogICAgcHJpbnQocGFzdGUoInRyYWluaW5nIHNldDoiLG5uMikpCiAgICBwcmludChwYXN0ZSgidGVzdCBzZXQ6IixubjEpKQogICAgCiAgICAjIGdldCB0aGUgZGF0YSwgbWVyZ2Ugc2FtcGxlcyBieSBiaWQgaWYgbmVjZXNzYXJ5CiAgICB5ID0gbWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0c1tbbm4xXV0kc2FtcGxlX2RhdGEKICAgIHggPSBtZXRhYm9sb21pY3NfcHJvY2Vzc2VkX2RhdGFzZXRzW1tubjJdXSRzYW1wbGVfZGF0YQogICAgeV9tZXRhID0gdW5pcXVlKG1ldGFib2xvbWljc19wcm9jZXNzZWRfZGF0YXNldHNbW25uMV1dJHNhbXBsZV9tZXRhX3BhcnNlZCkKICAgICMgIyByZW1vdmUgbWV0YWRhdGEgdmFyaWFibGVzIHdpdGggdG9vIG1hbnkgTkFzCiAgICAjIG5hX2NvdW50cyA9IGFwcGx5KGlzLm5hKHlfbWV0YSksMixzdW0pCiAgICAjIHlfbWV0YSA9IHlfbWV0YVssbmFfY291bnRzL25yb3coeV9tZXRhKSA9PSAwXQogICAgcm93bmFtZXMoeV9tZXRhKSA9IHlfbWV0YSRiaWQKICAgIGJpZF95ID0gbWVyZ2VkX2RtYXFjX2RhdGFbY29sbmFtZXMoeSksImJpZCJdCiAgICBiaWRfeCA9IG1lcmdlZF9kbWFxY19kYXRhW2NvbG5hbWVzKHgpLCJiaWQiXSAgICAKICAgICMgbWVyZ2Ugc2FtcGxlcyBmcm9tIHRoZSBzYW1lIEJJRAogICAgaWYobGVuZ3RoKHVuaXF1ZShiaWRfeCkpIT1sZW5ndGgoYmlkX3gpKXsKICAgICAgeCA9IGFnZ3JlZ2F0ZV9yZXBlYXRlZF9zYW1wbGVzKHgsYmlkX3gpCiAgICB9CiAgICBlbHNlewogICAgICBjb2xuYW1lcyh4KSA9IGJpZF94CiAgICB9CiAgICBpZihsZW5ndGgodW5pcXVlKGJpZF95KSkhPWxlbmd0aChiaWRfeSkpewogICAgICB5ID0gYWdncmVnYXRlX3JlcGVhdGVkX3NhbXBsZXMoeSxiaWRfeSkKICAgIH1lbHNlewogICAgICBjb2xuYW1lcyh5KSA9IGJpZF95CiAgICB9CiAgICAKICAgIGlmKG5jb2woeSk+MTAwMCl7bmV4dH0KICAgIAogICAgY292X2NvbHMgPSBpbnRlcnNlY3QoY29sbmFtZXMoeV9tZXRhKSwKICAgICAgICAgICAgICAgICAgICAgICAgICBzZXRkaWZmKGJpb3NwZWNfY29scywiYmlkIikpCiAgICBjb3ZzID0gYXMubWF0cml4KHlfbWV0YVtjb2xuYW1lcyh5KSxjb3ZfY29sc10pCiAgICAKICAgICMgcmVncmVzcyB0aGUgY292YXJpYXRlcyBvdXQgLSBzaW1wbGUgbGluZWFyIGFuYWx5c2lzCiAgICB5ID0gbG1fcmVncmVzc19vdXRfbWF0cml4KHQoeSksY292cykKICAgIAogICAgIyBQcmVwYXJlIHRoZSBpbnB1dCBmb3IgdGhlIE1MIHBhcnQKICAgIHNoYXJlZF9iaWRzID0gYXMuY2hhcmFjdGVyKGludGVyc2VjdChyb3duYW1lcyh5KSxjb2xuYW1lcyh4KSkpCiAgICB4ID0gdChhcy5tYXRyaXgoeFssc2hhcmVkX2JpZHNdKSkKICAgIHkgPSBhcy5tYXRyaXgoeVtzaGFyZWRfYmlkcyxdKQogICAgCiAgICAjIFJ1biB0aGUgcmVncmVzc2lvbnMKICAgIGZvbGRzID0gc2FtcGxlKHJlcCgxOm5mb2xkcywoMStucm93KHgpL25mb2xkcykpKVsxOm5yb3coeCldCiAgICBudW1GZWF0dXJlcyA9IG1pbihuY29sKHgpLDIwMDApCiAgICBwcmVkcyA9IGMoKTtyZWFsPWMoKQogICAgZm9yKGkgaW4gMTpuY29sKHkpKXsKICAgICAgeV9pID0geVssMV0KICAgICAgaV9wcmVkcyA9IGMoKTtpX3JlYWw9YygpCiAgICAgIGZvcihqIGluIDE6bmZvbGRzKXsKICAgICAgICBwcmludChqKQogICAgICAgIHRyX3ggPSB4W2ZvbGRzIT1qLF0KICAgICAgICB0cl95aSA9IHlfaVtmb2xkcyE9al0KICAgICAgICB0ZV94ID0geFtmb2xkcz09aixdCiAgICAgICAgdGVfeSA9IHlfaVtmb2xkcz09al0KICAgICAgICAjIHJhbmRvbSBmb3Jlc3QKICAgICAgICAjIG1vZGVsID0gcmFuZG9tRm9yZXN0KHRyX3lpLHg9dHJfeCxudHJlZSA9IDIwKQogICAgICAgICMgdGVfcHJlZHMgPSBwcmVkaWN0KG1vZGVsLG5ld2RhdGEgPSB0ZV94KQogICAgICAgIG1vZGVsID0gZmVhdHVyZV9zZWxlY3Rpb25fd3JhcHBlcih0cl94LHRyX3lpLAogICAgICAgICAgICAgICAgICAgY29lZmZfb2ZfdmFyLHJhbmRvbUZvcmVzdCwKICAgICAgICAgICAgICAgICAgIHRvcEsgPSBudW1GZWF0dXJlcyxudHJlZT01MCkKICAgICAgICB0ZV9wcmVkcyA9IHByZWRpY3QobW9kZWwsbmV3ZGF0YSA9IHRlX3gpCiAgICAgICAgaV9wcmVkcyA9IGMoaV9wcmVkcyx0ZV9wcmVkcykKICAgICAgICBpX3JlYWwgPSBjKGlfcmVhbCx0ZV95KQogICAgICB9CiAgICAgIHByZWRzID0gY2JpbmQocHJlZHMsaV9wcmVkcykKICAgICAgcmVhbCA9IGNiaW5kKHJlYWwsaV9yZWFsKQogICAgfQogICAgY3Vycm5hbWUgPSBwYXN0ZShubjEsbm4yLHNlcD0iOyIpCiAgICBwcmVkaWN0aW9uX2FuYWx5c2lzX3Jlc3VsdHNbW2N1cnJuYW1lXV0gPSBsaXN0KAogICAgICBwcmVkcyA9IHByZWRzLHJlYWw9cmVhbAogICAgKQogIH0KfQpuYW1lcyhwcmVkaWN0aW9uX2FuYWx5c2lzX3Jlc3VsdHMpCgpjb3ZfcHJlZGljdGlvbl9hbmFseXNpc19yZXN1bHRzID0gbGlzdCgpCmZvcihubjEgaW4gbmFtZXMobWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0cykpewogIHByaW50KG5uMSkKICB5ID0gbWV0YWJvbG9taWNzX3Byb2Nlc3NlZF9kYXRhc2V0c1tbbm4xXV0kc2FtcGxlX2RhdGEKICB5X3ZpYWxzID0gY29sbmFtZXMoeSkKICBiaWRfeSA9IG1lcmdlZF9kbWFxY19kYXRhW2NvbG5hbWVzKHkpLCJiaWQiXQogIGNvbG5hbWVzKHkpID0gYmlkX3kKICB5ID0gdChhcy5tYXRyaXgoeSkpCiAgaWYobmNvbCh5KT4xMDAwKXtuZXh0fQogIGNvdl9jb2xzID0gYygiYW5pbWFsLnJlZ2lzdHJhdGlvbi5zZXgiLAogICAgICAgICAgICAgImFjdXRlLnRlc3Qud2VpZ2h0IiwKICAgICAgICAgICAgICJhY3V0ZS50ZXN0LmRpc3RhbmNlIiwKICAgICAgICAgICAgICJhbmltYWwua2V5LnRpbWVwb2ludCIpCiAgY292cyA9IG1lcmdlZF9kbWFxY19kYXRhW3lfdmlhbHMsY292X2NvbHNdCiAgeCA9IGNvdnMKICAKICAjIFJ1biB0aGUgcmVncmVzc2lvbnMKICBmb2xkcyA9IHNhbXBsZShyZXAoMTpuZm9sZHMsKDErbnJvdyh4KS9uZm9sZHMpKSlbMTpucm93KHgpXQogIG51bUZlYXR1cmVzID0gbWluKG5jb2woeCksMjAwMCkKICBwcmVkcyA9IGMoKTtyZWFsPWMoKQogIGZvcihpIGluIDE6bmNvbCh5KSl7CiAgICB5X2kgPSB5WywxXQogICAgaV9wcmVkcyA9IGMoKTtpX3JlYWw9YygpCiAgICBmb3IoaiBpbiAxOm5mb2xkcyl7CiAgICAgIHByaW50KGopCiAgICAgIHRyX3ggPSB4W2ZvbGRzIT1qLF0KICAgICAgdHJfeWkgPSB5X2lbZm9sZHMhPWpdCiAgICAgIHRlX3ggPSB4W2ZvbGRzPT1qLF0KICAgICAgdGVfeSA9IHlfaVtmb2xkcz09al0KICAgICAgIyByYW5kb20gZm9yZXN0CiAgICAgIG1vZGVsID0gcmFuZG9tRm9yZXN0KHRyX3lpLHg9dHJfeCxudHJlZSA9IDIwKQogICAgICB0ZV9wcmVkcyA9IHByZWRpY3QobW9kZWwsbmV3ZGF0YSA9IHRlX3gpCiAgICAgIGlfcHJlZHMgPSBjKGlfcHJlZHMsdGVfcHJlZHMpCiAgICAgIGlfcmVhbCA9IGMoaV9yZWFsLHRlX3kpCiAgICB9CiAgICBwcmVkcyA9IGNiaW5kKHByZWRzLGlfcHJlZHMpCiAgICByZWFsID0gY2JpbmQocmVhbCxpX3JlYWwpCiAgfQogIGNvdl9wcmVkaWN0aW9uX2FuYWx5c2lzX3Jlc3VsdHNbW25uMV1dID0gbGlzdCgKICAgICAgcHJlZHMgPSBwcmVkcyxyZWFsPXJlYWwKICAgICkKfQoKcmVzdWx0c19tZXRyaWNzID0gYygpCmZvcihubiBpbiBuYW1lcyhwcmVkaWN0aW9uX2FuYWx5c2lzX3Jlc3VsdHMpKXsKICBwcmVkcyA9IHByZWRpY3Rpb25fYW5hbHlzaXNfcmVzdWx0c1tbbm5dXSRwcmVkcwogIHJlYWwgPSBwcmVkaWN0aW9uX2FuYWx5c2lzX3Jlc3VsdHNbW25uXV0kcmVhbAogIHJob3MxID0gZGlhZyhjb3IocHJlZHMscmVhbCkpCiAgcmhvczIgPSBkaWFnKGNvcihwcmVkcyxyZWFsLG1ldGhvZD0ic3BlYXJtYW4iKSkKICBTRXMgPSAocHJlZHMtcmVhbCleMgogIG1zZSA9IG1lYW4oU0VzKQogIG5vcm1TRXMgPSBTRXMgLyBhcHBseShyZWFsLDIsc2QpCiAgY3Vycl9zY29yZXMgPSBjKG1lYW4ocmhvczEpLG1lYW4ocmhvczIpLG1pbihyaG9zMSksbWluKHJob3MyKSwKICAgICAgICAgICAgICAgICAgbXNlLG1lYW4obm9ybVNFcykpCiAgbmFtZXMoY3Vycl9zY29yZXMpID0gYygibWVhbl9yaG8iLCJtZWFuX3NwZWFybWFuX3JobyIsIm1pbl9yaG8iLCJtaW5fc3BlYXJtYW5fcmhvIiwKICAgICAgICAgICAgICAgICAgICAgICAgICJNU0UiLCJtZWFuIE1TRS9TRCIpCiAgcmVzdWx0c19tZXRyaWNzID0gcmJpbmQocmVzdWx0c19tZXRyaWNzLAogICAgICAgICAgICAgICAgICAgICAgICAgIGN1cnJfc2NvcmVzKQogIHJvd25hbWVzKHJlc3VsdHNfbWV0cmljcylbbnJvdyhyZXN1bHRzX21ldHJpY3MpXSA9IG5uCn0KCmNvdl9yZXN1bHRzX21ldHJpY3MgPSBjKCkKZm9yKG5uIGluIG5hbWVzKGNvdl9wcmVkaWN0aW9uX2FuYWx5c2lzX3Jlc3VsdHMpKXsKICBwcmVkcyA9IGNvdl9wcmVkaWN0aW9uX2FuYWx5c2lzX3Jlc3VsdHNbW25uXV0kcHJlZHMKICByZWFsID0gY292X3ByZWRpY3Rpb25fYW5hbHlzaXNfcmVzdWx0c1tbbm5dXSRyZWFsCiAgcmhvczEgPSBkaWFnKGNvcihwcmVkcyxyZWFsKSkKICByaG9zMiA9IGRpYWcoY29yKHByZWRzLHJlYWwsbWV0aG9kPSJzcGVhcm1hbiIpKQogIFNFcyA9IChwcmVkcy1yZWFsKV4yCiAgbXNlID0gbWVhbihTRXMpCiAgbm9ybVNFcyA9IFNFcyAvIGFwcGx5KHJlYWwsMixzZCkKICBjdXJyX3Njb3JlcyA9IGMobWVhbihyaG9zMSksbWVhbihyaG9zMiksbWluKHJob3MxKSxtaW4ocmhvczIpLAogICAgICAgICAgICAgICAgICBtc2UsbWVhbihub3JtU0VzKSkKICBuYW1lcyhjdXJyX3Njb3JlcykgPSBjKCJtZWFuX3JobyIsIm1lYW5fc3BlYXJtYW5fcmhvIiwibWluX3JobyIsIm1pbl9zcGVhcm1hbl9yaG8iLAogICAgICAgICAgICAgICAgICAgICAgICAgIk1TRSIsIm1lYW4gTVNFL1NEIikKICBjb3ZfcmVzdWx0c19tZXRyaWNzID0gcmJpbmQoY292X3Jlc3VsdHNfbWV0cmljcywKICAgICAgICAgICAgICAgICAgICAgICAgICBjdXJyX3Njb3JlcykKICByb3duYW1lcyhjb3ZfcmVzdWx0c19tZXRyaWNzKVtucm93KGNvdl9yZXN1bHRzX21ldHJpY3MpXSA9IG5uCn0KCiMgU29tZSBib3hwbG90cwpwcmVkX3RhcmdldHMgPSBzYXBwbHkobmFtZXMocHJlZGljdGlvbl9hbmFseXNpc19yZXN1bHRzKSxmdW5jdGlvbih4KQogIHN0cnNwbGl0KHgsc3BsaXQgPSAiOyIpW1sxXV1bMV0pCnRhcmdldCA9ICJsaXZlcl9wb3dkZXIsbWV0YWJfdF90Y2EsbmFtZWQiIApjdXJyX3JlcyA9IHVuaXF1ZShyZXN1bHRzX21ldHJpY3NbcHJlZF90YXJnZXRzID09IHRhcmdldCxdKQpyb3duYW1lcyhjdXJyX3JlcykgPSBnc3ViKHRhcmdldCwiIixyb3duYW1lcyhjdXJyX3JlcykpCnJvd25hbWVzKGN1cnJfcmVzKSA9IGdzdWIoIjsiLCIiLHJvd25hbWVzKGN1cnJfcmVzKSkKcm93bmFtZXMoY3Vycl9yZXMpW3Jvd25hbWVzKGN1cnJfcmVzKT09IiJdID0gdGFyZ2V0CmNvdl9iYXNlbGluZSA9IGNvdl9yZXN1bHRzX21ldHJpY3NbdGFyZ2V0LF0KCnBhcihtYXIgPSBjKDgsNCwyLDIpKQpjb2xzID0gcmVwKCJibHVlIixucm93KGN1cnJfcmVzKSkKY29sc1tyb3duYW1lcyhjdXJyX3Jlcyk9PXRhcmdldF0gPSAiYmxhY2siCnBsdCA9IGJhcnBsb3QoY3Vycl9yZXNbLDRdLGJlc2lkZSA9IFQseGF4dD0ibiIsbGVnZW5kPUYsCiAgICAgICAgICAgICAgeWxhYj0iTWluIHJobyAoU3BlYXJtYW4pIiwKICAgICAgICAgICAgICB5bGltID0gYygwLjUsMSkseHBkPUYsY29sPWNvbHMsCiAgICAgICAgICAgICAgbWFpbiA9IHRhcmdldCkKdGV4dChwbHQsIHBhcigidXNyIilbM10sIGxhYmVscyA9IHJvd25hbWVzKGN1cnJfcmVzKSwgCiAgICAgc3J0ID0gNDUsIGFkaiA9IGMoMS4xLDEuMSksIHhwZCA9IFQsIGNleD0wLjYpCmFibGluZShoPWNvdl9iYXNlbGluZVs0XSxsdHk9Mixsd2Q9Mixjb2w9InJlZCIpCgpwYXIobWFyID0gYyg4LDQsMiwyKSkKcGx0ID0gYmFycGxvdChjdXJyX3Jlc1ssNl0sYmVzaWRlID0gVCx4YXh0PSJuIixsZWdlbmQ9RiwKICAgICAgICAgICAgICB5bGFiPSJNZWFuIHN0YW5kYXJkaXplZCBTRSIseHBkPUYsCiAgICAgICAgICAgICAgbWFpbiA9IHRhcmdldCxjb2w9Y29scykKdGV4dChwbHQsIHBhcigidXNyIilbM10sIGxhYmVscyA9IHJvd25hbWVzKGN1cnJfcmVzKSwgCiAgICAgc3J0ID0gNDUsIGFkaiA9IGMoMS4xLDEuMSksIHhwZCA9IFQsIGNleD0wLjYpCmFibGluZShoPWNvdl9iYXNlbGluZVs2XSxsdHk9Mixsd2Q9Mixjb2w9InJlZCIpCgojIHByZWRzID0gYygpO3JlYWw9YygpCiMgZm9yKGogaW4gMTpuZm9sZHMpewojICAgdHJfeCA9IHhbZm9sZHMhPWosXQojICAgdHJfeSA9IHlbZm9sZHMhPWosXQojICAgdGVfeCA9IHhbZm9sZHM9PWosXQojICAgdGVfeSA9IHlbZm9sZHM9PWosXQojICAgbW9kZWwgPSBNVExfd3JhcHBlcih0cl94LHRyX3ksdHlwZT0iUmVncmVzc2lvbiIsIFJlZ3VsYXJpemF0aW9uPSJMMjEiKQojICAgdGVfcHJlZHMgPSBwcmVkaWN0KG1vZGVsLHRlX3gpCiMgICByZWFsID0gcmJpbmQocmVhbCx0ZV95KQojICAgcHJlZHMgPSByYmluZChwcmVkcyx0ZV9wcmVkcykKIyB9CiMgZGlhZyhjb3IocHJlZHMscmVhbCkpCgojIFVzaW5nIFBMUyByZWdyZXNzaW9uCiMgbGlicmFyeShwbHMpCiMgcGxzX21vZGVsID0gcGxzcih5fngsbmNvbXAgPSA1LHZhbGlkYXRpb249IkxPTyIpCiMgZXZhbCA9IE1TRVAocGxzX21vZGVsKQojIAojIHlfcGNhID0gcHJjb21wKHkpCiMgcGxvdCh5X3BjYSkKIyBleHBsYWluZWRfdmFyID0geV9wY2Ekc2Rldl4yL3N1bSh5X3BjYSRzZGV2XjIpCiMgeV9wY2FfbWF0cml4ID0geV9wY2EkeFssMToxMF0KIyAKIyAjIHJlZ3Jlc3Mgb3V0IHNleCwgd2VpZ2h0CiMgCiMgZ2V0X2V4cGxhaW5lZF92YXJpYW5jZV91c2luZ19QQ0EoeCx5KQojIHggPSBhcHBseSh4LDIscmVncmVzc19vdXQsY292cz1jb3ZzKQojIHkgPSBhcHBseSh5LDIscmVncmVzc19vdXQsY292cz1jb3ZzKQojIGdldF9leHBsYWluZWRfdmFyaWFuY2VfdXNpbmdfUENBKHgseSkKCgpgYGAKCiMjIENvbXBhcmlzb24gb2YgY292YXJpYW5jZSBtYXRyaWNlcwoKYGBge3J9CgpDViA8LWZ1bmN0aW9uKHgpewogIHJldHVybihzZCh4KS9tZWFuKHgpKQp9CgojIEdldCBhbGwgY29ycmVsYXRpb24gYW5kIGNvdmFyaWFuY2UgbWF0cmljZXMKeSA9IG1ldGFib2xvbWljc19wYXJzZWRfZGF0YXNldHNbWzFdXSRzYW1wbGVfZGF0YSAjIGFuY2hvciBhbGwgZGF0YXNldHMgYnkgdGhlc2UgaWRzCnlfdmlhbHMgPSBjb2xuYW1lcyh5KQpiaWRfeSA9IGFzLmNoYXJhY3RlcihtZXJnZWRfZG1hcWNfZGF0YVtjb2xuYW1lcyh5KSwiYmlkIl0pCmNvdl9jb2xzID0gYygiYW5pbWFsLnJlZ2lzdHJhdGlvbi5zZXgiLAogICAgICAgICAgICAgImFjdXRlLnRlc3Qud2VpZ2h0IiwKICAgICAgICAgICAgICJhY3V0ZS50ZXN0LmRpc3RhbmNlIikKY292cyA9IG1lcmdlZF9kbWFxY19kYXRhW3lfdmlhbHMsY292X2NvbHNdCgpjb3ZfbWF0cmljZXMgPSBsaXN0KCkKY29yX21hdHJpY2VzID0gbGlzdCgpCmZvcihubiBpbiBuYW1lcyhtZXRhYm9sb21pY3NfcGFyc2VkX2RhdGFzZXRzKSl7CiAgeCA9IG1ldGFib2xvbWljc19wYXJzZWRfZGF0YXNldHNbW25uXV0kc2FtcGxlX2RhdGEKICBiaWRfeCA9IGFzLmNoYXJhY3RlcihtZXJnZWRfZG1hcWNfZGF0YVtjb2xuYW1lcyh4KSwiYmlkIl0pCiAgcHJpbnQoc3VtKCFpcy5lbGVtZW50KGJpZF95LHNldD1iaWRfeCkpKSAjIHNob3VsZCBiZSB6ZXJvCiAgY29sbmFtZXMoeCkgPSBiaWRfeAogIHggPSB4WyxiaWRfeV0KICAjIHggPSBhcHBseSh4LDEscmVncmVzc19vdXQsY292cz1jb3ZzKQogICMgeCA9IHQoeCkKICAjIGN2cyA9IGFwcGx5KHgsMSxDVikKICAjIHByaW50KHRhYmxlKGN2cz4wLjUpKQogICMgeCA9IHhbY3ZzPjAuNSxdCiAgIyBwY2F4ID0gdChwcmNvbXAodCh4KSxzY2FsZS4gPSBGKSR4WywxOjEwXSkKICBjb3ZfbWF0cmljZXNbW25uXV0gPSBjb3YoeCkKICBjb3JfbWF0cmljZXNbW25uXV0gPSBjb3IoeCkKfQpzYXBwbHkoY29yX21hdHJpY2VzLGRpbSkKbGlicmFyeSgnZXZvbHFnJyk7bGlicmFyeShjb3JycGxvdCkKbWFudGVsX3Rlc3RzPSBNYW50ZWxDb3IoY29yX21hdHJpY2VzKQptY29ycz0gTWF0cml4Q29yKGNvcl9tYXRyaWNlcykKbWNvcnMgPSBhcy5tYXRyaXgoZm9yY2VTeW1tZXRyaWMobWNvcnMsdXBsbz0iTCIpKQptYW50ZWxfdGVzdHMgPSBhcy5tYXRyaXgoCiAgZm9yY2VTeW1tZXRyaWMobWFudGVsX3Rlc3RzJHByb2JhYmlsaXRpZXMsdXBsbyA9ICJMIikpCiMgZGlhZyhtYW50ZWxfdGVzdHMpPTAKb3JkID0gY29ycnBsb3QodChtY29ycyksb3JkZXI9ImhjbHVzdCIpCm9yZCA9IHJvd25hbWVzKG9yZCkKY29ycnBsb3QobWNvcnNbb3JkLG9yZF0scC5tYXQ9bWFudGVsX3Rlc3RzW29yZCxvcmRdLAogICAgICAgICBpbnNpZyA9ICJsYWJlbF9zaWciLG1ldGhvZD0ic2hhZGUiLAogICAgICAgICBzaWcubGV2ZWwgPSBjKC4wMDAxLCAuMDAxLCAuMDEpLAogICAgICAgICBwY2guY2V4ID0gLjksIHBjaC5jb2wgPSAiYmxhY2siKQojIHJzX3JlcyA9IFJhbmRvbVNrZXdlcnMoY29yX21hdHJpY2VzKQojIGNvcnJwbG90KHJzX3JlcyRjb3JyZWxhdGlvbnMsCiMgICAgICAgICAgcC5tYXQ9bWFudGVsX3Rlc3RzJHByb2JhYmlsaXRpZXMsdHlwZT0ibG93ZXIiLAojICAgICAgICAgIGluc2lnID0gImxhYmVsX3NpZyIsbWV0aG9kPSJzaGFkZSIsCiMgICAgICAgICAgc2lnLmxldmVsID0gYyguMDAwMSwgLjAwMSwgLjAxKSwKIyAgICAgICAgICBwY2guY2V4ID0gLjksIHBjaC5jb2wgPSAiYmxhY2siLHRsLmNleD0wLjYpCgpsaWJyYXJ5KHBzeWNoKQoKcjEgPSBjb3JfbWF0cmljZXNbWzFdXQpyMiA9IGNvcl9tYXRyaWNlc1tbOF1dCmNvcihyMVtsb3dlci50cmkocjEpXSxyMltsb3dlci50cmkocjIpXSkKbWFudGVsX3Rlc3RzWzEsOF0KbWFudGVsX3Rlc3RzWzgsMV0KCnRocmVzaG9sZF9jb21wID0gYXBwbHlfZnVuY3Rpb25fb25fcGFpcnMoY29yX21hdHJpY2VzLAogICAgICAgICAgZnVuY3Rpb24oeCx5KXN1bSh4PjAuNyZ5PjAuNykpCmNvcnJwbG90KHRocmVzaG9sZF9jb21wLGlzLmNvcnIgPSBGLG9yZGVyPSJoY2x1c3QiKQoKdGhyZXNob2xkX2NvbXAgPSBhcHBseV9mdW5jdGlvbl9vbl9wYWlycyhjb3JfbWF0cmljZXMsCiAgICAgICAgIGZ1bmN0aW9uKHgseSltZWFuKGRpYWcoY29yKHgseSkpKSkKY29ycnBsb3QodGhyZXNob2xkX2NvbXAsaXMuY29yciA9IEYsb3JkZXI9ImhjbHVzdCIpCgpgYGAKCiMgSW50ZWdyYXJpb24gd2l0aCBSTkFzZXEKCmBgYHtyfQpybmFzZXFfZGF0YV9mb3JfZGlmZnRlc3RzID0gbG9hZF9mcm9tX2J1Y2tldCgKICAicm5hc2VxX2RhdGFfZm9yX2RpZmZ0ZXN0cy5SRGF0YSIsCiAgImdzOi8vYmljX2RhdGFfYW5hbHlzaXMvcGFzczFhL3JuYXNlcS8iCilbWzFdXQptZXRfdnNfcm5hc2VxX2NvcnMgPSBjKCkKZm9yKG5uIGluIG5hbWVzKHJuYXNlcV9kYXRhX2Zvcl9kaWZmdGVzdHMpKXsKICBkID0gcm5hc2VxX2RhdGFfZm9yX2RpZmZ0ZXN0c1tbbm5dXSRmcGttcwogIGNvdnMgPSBybmFzZXFfZGF0YV9mb3JfZGlmZnRlc3RzW1tubl1dJGRtYXFjX21ldGFbLGNvdl9jb2xzXQogIAogIGN2cyA9IGFwcGx5KGQsMSxDVikKICBwcmludCh0YWJsZShjdnM+MC4yNSkpCiAgeCA9IGRbY3ZzPjAuMjUsXQogICMgeCA9IGFwcGx5KHgsMSxyZWdyZXNzX291dCxjb3ZzPWNvdnMpCiAgIyB4ID0gdCh4KQogIAogIGJpZF94ID0gc2FwcGx5KGNvbG5hbWVzKHgpLHN1YnN0ciwxLDUpCiAgcHJpbnQoc3VtKCFpcy5lbGVtZW50KGJpZF95LHNldD1iaWRfeCkpKSAjIHNob3VsZCBiZSB6ZXJvCiAgY29sbmFtZXMoeCkgPSBiaWRfeAogIHggPSB4WyxiaWRfeV0KICAKICBjdXJyX2NvcnMgPSBjb3IoeCkKICB2ID0gYygpCiAgZm9yKG5uMiBpbiBuYW1lcyhjb3JfbWF0cmljZXMpKXsKICAgIHZbbm4yXSA9IE1hdHJpeENvcihjdXJyX2NvcnMsY29yX21hdHJpY2VzW1tubjJdXSkKICB9CiAgbWV0X3ZzX3JuYXNlcV9jb3JzID0gcmJpbmQobWV0X3ZzX3JuYXNlcV9jb3JzLHYpCiAgcm93bmFtZXMobWV0X3ZzX3JuYXNlcV9jb3JzKVtucm93KG1ldF92c19ybmFzZXFfY29ycyldID0gbm4KfQpsaWJyYXJ5KGdwbG90cykKaGVhdG1hcC4yKHQobWV0X3ZzX3JuYXNlcV9jb3JzKSwKICAgICAgICAgIHRyYWNlPSJub25lIixzY2FsZT1OVUxMLG1hcj1jKDE1LDE1KSwKICAgICAgICAgIGtleS54bGFiID0gIkNvcnJlbGF0aW9uIiwKICAgICAgICAgIGNvbD1yZWRibHVlKDIwMCksIGJyZWFrcz1zZXEoLTEsMSwwLjAxKSkKCmBgYAoKIyBMb29raW5nIGF0IG1ldGFib2xpdGUgb3ZlcmxhcAoKYGBge3IsZmlnLmFsaWduPSJjZW50ZXIiLG91dC5oZWlnaHQ9JzUwJScsb3V0LndpZHRoPSc4MCUnLG1lc3NhZ2U9RkFMU0Usd2FybmluZz1GQUxTRX0KbWV0YWJvbGl0ZV9zZXRzID0gbGlzdCgpCmZvcihubiBpbiBuYW1lcyhtZXRhYm9sb21pY3NfcGFyc2VkX2RhdGFzZXRzKSl7CiAgYW4gPSBtZXRhYm9sb21pY3NfcGFyc2VkX2RhdGFzZXRzW1tubl1dJHJvd19hbm5vdAogIG5hbWVfY29scyA9IGNvbG5hbWVzKGFuKVtncmVwbCgiX25hbWUiLGNvbG5hbWVzKGFuKSxpZ25vcmUuY2FzZSA9IFQpXQogIG1ldGFib2xpdGVfc2V0c1tbbm5dXSA9IHVuaXF1ZSh0b2xvd2VyKGFuWyxuYW1lX2NvbHNbMV1dKSkKfQpvdmVybGFwX21hdHJpeCA9IGFwcGx5X2Z1bmN0aW9uX29uX3BhaXJzKG1ldGFib2xpdGVfc2V0cywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmdW5jdGlvbih4LHkpbGVuZ3RoKGludGVyc2VjdCh4LHkpKSkKY29ycnBsb3QobG9nKG92ZXJsYXBfbWF0cml4KzEsMTApLGlzLmNvcnIgPSBGKQoKCnNpdGVfY29yX2NvbXBhcmlzb24gPSBhcHBseV9mdW5jdGlvbl9vbl9wYWlycygKICBtZXRhYm9sb21pY3NfcGFyc2VkX2RhdGFzZXRzLAogIGNvbXBhcmVfdHdvX21ldF9zaXRlcyxwcmludF9wcm9ncmVzcyA9IFQsCiAgbWVyZ2VkX2RtYXFjX2RhdGEgPSBtZXJnZWRfZG1hcWNfZGF0YSxzYW1wbGVzaXplPTUwMAopCgpzaXRlX2Nvcl9jb21wYXJpc29uW2lzLm5hKHNpdGVfY29yX2NvbXBhcmlzb24pXT0wCmNvcnJwbG90KHNpdGVfY29yX2NvbXBhcmlzb24sb3JkZXI9ImhjbHVzdCIpCgpjb21wYXJlX3R3b19tZXRfc2l0ZXM8LWZ1bmN0aW9uKHgseSxtZXJnZWRfZG1hcWNfZGF0YSxzYW1wbGVzaXplPTEwMCwuLi4pewogIGFuX3ggPSB4JHJvd19hbm5vdAogIG5hbWVfY29sc3ggPSBjb2xuYW1lcyhhbl94KVtncmVwbCgiX25hbWUiLGNvbG5hbWVzKGFuX3gpLGlnbm9yZS5jYXNlID0gVCldCiAgYW5feSA9IHkkcm93X2Fubm90CiAgbmFtZV9jb2xzeSA9IGNvbG5hbWVzKGFuX3kpW2dyZXBsKCJfbmFtZSIsY29sbmFtZXMoYW5feSksaWdub3JlLmNhc2UgPSBUKV0KICAKICBuYW1lc194ID0gYXMuY2hhcmFjdGVyKGFuX3hbLG5hbWVfY29sc3hbMV1dKQogIG5hbWVzX3kgPSBhcy5jaGFyYWN0ZXIoYW5feVssbmFtZV9jb2xzeVsxXV0pCiAgc2hhcmVkID0gaW50ZXJzZWN0KG5hbWVzX3gsbmFtZXNfeSkKICBzaGFyZWQgPSBzZXRkaWZmKHNoYXJlZCxjKE5BLCIiKSkKICAKICBteCA9IHgkc2FtcGxlX2RhdGEKICBteSA9IHkkc2FtcGxlX2RhdGEKICAKICBpZihhbnkodGFibGUobmFtZXNfeCk+MSkpewogICAgbXggPSBhcHBseShteCwyLGZ1bmN0aW9uKHgseSl0YXBwbHkoeCxJTkRFWD15LEZVTj1tZWFuLG5hLnJtPVQpLHk9bmFtZXNfeCkKICB9CiAgZWxzZXsKICAgIHJvd25hbWVzKG14KSA9IGFzLmNoYXJhY3RlcihuYW1lc194KQogIH0KICBpZihhbnkodGFibGUobmFtZXNfeSk+MSkpewogICAgbXkgPSBhcHBseShteSwyLGZ1bmN0aW9uKHgseSl0YXBwbHkoeCxJTkRFWD15LEZVTj1tZWFuLG5hLnJtPVQpLHk9bmFtZXNfeSkKICB9CiAgZWxzZXsKICAgIHJvd25hbWVzKG15KSA9IGFzLmNoYXJhY3RlcihuYW1lc195KQogIH0KICAKICBiaWRfeCA9IGFzLmNoYXJhY3RlcihtZXJnZWRfZG1hcWNfZGF0YVtjb2xuYW1lcyhteCksImJpZCJdKQogIGJpZF95ID0gYXMuY2hhcmFjdGVyKG1lcmdlZF9kbWFxY19kYXRhW2NvbG5hbWVzKG15KSwiYmlkIl0pCiAgaWYoc3VtKCFpcy5lbGVtZW50KGJpZF95LHNldD1iaWRfeCkpPjApewogICAgc3RvcCgiQklEcyBkbyBub3QgbWF0Y2ggYmV0d2VlbiB4IGFuZCB5IikKICB9ICMgc2hvdWxkIGJlIHplcm8KICBjb2xuYW1lcyhteCkgPSBiaWRfeAogIGNvbG5hbWVzKG15KSA9IGJpZF95CiAgaWYobGVuZ3RoKHNoYXJlZCk+c2FtcGxlc2l6ZSl7CiAgICBzaGFyZWQgPSBzYW1wbGUoc2hhcmVkKVsxOnNhbXBsZXNpemVdCiAgfQogIG14ID0gbXhbc2hhcmVkLGJpZF95XQogIG15ID0gbXlbc2hhcmVkLF0KICBteCA9IGFzLm1hdHJpeChteCk7bXkgPSBhcy5tYXRyaXgobXkpCiAgaWYobnJvdyhteCk8Mil7cmV0dXJuKE5BKX0KICBpZihsZW5ndGgoc2hhcmVkKTw9MTAwKXsKICAgIGNvcnJzID0gZGlhZyhjb3IodChteCksdChteSkpKQogIH0KICBlbHNlewogICAgY29ycnMgPSBjKCkKICAgIGZvcihpIGluIDE6bnJvdyhteCkpewogICAgICAjIHByaW50KG1lYW4obXhbaSxdKSkKICAgICAgY29ycnNbcm93bmFtZXMobXgpW2ldXSA9IGNvcihteFtpLF0sbXlbaSxdKQogICAgfQogIH0KICByZXR1cm4obWVhbihjb3JycykpCn0KCmBgYAoKCgoKCgoKCgoKCgoK